Merge pull request #151 from goccy/feature/support-decoding-of-tag-an…

…d-map-key

Support decoding of canonical yaml

ast/ast.go

@@ -42,6 +42,8 @@ const (
 	LiteralType
 	// MappingType type identifier for mapping node
 	MappingType
+	// MappingKeyType type identifier for mapping key node
+	MappingKeyType
 	// MappingValueType type identifier for mapping value node
 	MappingValueType
 	// SequenceType type identifier for sequence node
@@ -85,6 +87,8 @@ func (t NodeType) String() string {
 		return "Literal"
 	case MappingType:
 		return "Mapping"
+	case MappingKeyType:
+		return "MappingKey"
 	case MappingValueType:
 		return "MappingValue"
 	case SequenceType:
@@ -329,6 +333,13 @@ func Mapping(tk *token.Token, isFlowStyle bool) *MappingNode {
 	}
 }
 
+// MappingKey create node for map key ( '?' ).
+func MappingKey(tk *token.Token) *MappingKeyNode {
+	return &MappingKeyNode{
+		Start: tk,
+	}
+}
+
 // Sequence create node for sequence
 func Sequence(tk *token.Token, isFlowStyle bool) *SequenceNode {
 	return &SequenceNode{
@@ -878,6 +889,48 @@ func (n *MappingNode) MapRange() *MapNodeIter {
 	}
 }
 
+// MappingKeyNode type of tag node
+type MappingKeyNode struct {
+	Comment *token.Token // position of Comment ( `#comment` )
+	Start   *token.Token
+	Value   Node
+}
+
+// Type returns MappingKeyType
+func (n *MappingKeyNode) Type() NodeType { return MappingKeyType }
+
+// GetToken returns token instance
+func (n *MappingKeyNode) GetToken() *token.Token {
+	return n.Start
+}
+
+// GetComment returns comment token instance
+func (n *MappingKeyNode) GetComment() *token.Token {
+	return n.Comment
+}
+
+// AddColumn add column number to child nodes recursively
+func (n *MappingKeyNode) AddColumn(col int) {
+	n.Start.AddColumn(col)
+	if n.Value != nil {
+		n.Value.AddColumn(col)
+	}
+}
+
+// SetComment set comment token
+func (n *MappingKeyNode) SetComment(tk *token.Token) error {
+	if tk.Type != token.CommentType {
+		return ErrInvalidTokenType
+	}
+	n.Comment = tk
+	return nil
+}
+
+// String tag to text
+func (n *MappingKeyNode) String() string {
+	return fmt.Sprintf("%s %s", n.Start.Value, n.Value.String())
+}
+
 // MappingValueNode type of mapping value
 type MappingValueNode struct {
 	Comment *token.Token // position of Comment ( `#comment` )
@@ -1302,10 +1355,14 @@ func Walk(v Visitor, node Node) {
 	case *BoolNode:
 	case *InfinityNode:
 	case *NanNode:
+	case *TagNode:
+		Walk(v, n.Value)
 	case *MappingNode:
 		for _, value := range n.Values {
 			Walk(v, value)
 		}
+	case *MappingKeyNode:
+		Walk(v, n.Value)
 	case *MappingValueNode:
 		Walk(v, n.Key)
 		Walk(v, n.Value)

decode.go

@@ -101,13 +101,24 @@ func (d *Decoder) mergeValueNode(value ast.Node) ast.Node {
 	return value
 }
 
+func (d *Decoder) mapKeyNodeToString(node ast.Node) string {
+	key := d.nodeToValue(node)
+	if key == nil {
+		return "null"
+	}
+	if k, ok := key.(string); ok {
+		return k
+	}
+	return fmt.Sprint(key)
+}
+
 func (d *Decoder) setToMapValue(node ast.Node, m map[string]interface{}) {
 	switch n := node.(type) {
 	case *ast.MappingValueNode:
 		if n.Key.Type() == ast.MergeKeyType {
 			d.setToMapValue(d.mergeValueNode(n.Value), m)
 		} else {
-			key := n.Key.GetToken().Value
+			key := d.mapKeyNodeToString(n.Key)
 			m[key] = d.nodeToValue(n.Value)
 		}
 	case *ast.MappingNode:
@@ -123,7 +134,7 @@ func (d *Decoder) setToOrderedMapValue(node ast.Node, m *MapSlice) {
 		if n.Key.Type() == ast.MergeKeyType {
 			d.setToOrderedMapValue(d.mergeValueNode(n.Value), m)
 		} else {
-			key := n.Key.GetToken().Value
+			key := d.mapKeyNodeToString(n.Key)
 			*m = append(*m, MapItem{Key: key, Value: d.nodeToValue(n.Value)})
 		}
 	case *ast.MappingNode:
@@ -150,17 +161,24 @@ func (d *Decoder) nodeToValue(node ast.Node) interface{} {
 	case *ast.NanNode:
 		return n.GetValue()
 	case *ast.TagNode:
-		switch n.Start.Value {
+		switch token.ReservedTagKeyword(n.Start.Value) {
 		case token.TimestampTag:
 			t, _ := d.castToTime(n.Value)
 			return t
+		case token.IntegerTag:
+			i, _ := strconv.Atoi(fmt.Sprint(d.nodeToValue(n.Value)))
+			return i
 		case token.FloatTag:
 			return d.castToFloat(d.nodeToValue(n.Value))
 		case token.NullTag:
 			return nil
 		case token.BinaryTag:
 			b, _ := base64.StdEncoding.DecodeString(d.nodeToValue(n.Value).(string))
 			return b
+		case token.StringTag:
+			return d.nodeToValue(n.Value)
+		case token.MappingTag:
+			return d.nodeToValue(n.Value)
 		}
 	case *ast.AnchorNode:
 		anchorName := n.Name.GetToken().Value
@@ -173,6 +191,8 @@ func (d *Decoder) nodeToValue(node ast.Node) interface{} {
 		return d.nodeToValue(node)
 	case *ast.LiteralNode:
 		return n.Value.GetValue()
+	case *ast.MappingKeyNode:
+		return d.nodeToValue(n.Value)
 	case *ast.MappingValueNode:
 		if n.Key.Type() == ast.MergeKeyType {
 			value := d.mergeValueNode(n.Value)
@@ -185,7 +205,7 @@ func (d *Decoder) nodeToValue(node ast.Node) interface{} {
 			d.setToMapValue(value, m)
 			return m
 		}
-		key := n.Key.GetToken().Value
+		key := d.mapKeyNodeToString(n.Key)
 		if d.useOrderedMap {
 			return MapSlice{{Key: key, Value: d.nodeToValue(n.Value)}}
 		}
@@ -221,6 +241,10 @@ func (d *Decoder) resolveAlias(node ast.Node) ast.Node {
 		for idx, value := range n.Values {
 			n.Values[idx] = d.resolveAlias(value).(*ast.MappingValueNode)
 		}
+	case *ast.TagNode:
+		n.Value = d.resolveAlias(n.Value)
+	case *ast.MappingKeyNode:
+		n.Value = d.resolveAlias(n.Value)
 	case *ast.MappingValueNode:
 		if n.Key.Type() == ast.MergeKeyType && n.Value.Type() == ast.AliasType {
 			value := d.resolveAlias(n.Value)

decode_test.go

@@ -981,21 +981,23 @@ c:
 		},
 	}
 	for _, test := range tests {
-		buf := bytes.NewBufferString(test.source)
-		dec := yaml.NewDecoder(buf)
-		typ := reflect.ValueOf(test.value).Type()
-		value := reflect.New(typ)
-		if err := dec.Decode(value.Interface()); err != nil {
-			if err == io.EOF {
-				continue
+		t.Run(test.source, func(t *testing.T) {
+			buf := bytes.NewBufferString(test.source)
+			dec := yaml.NewDecoder(buf)
+			typ := reflect.ValueOf(test.value).Type()
+			value := reflect.New(typ)
+			if err := dec.Decode(value.Interface()); err != nil {
+				if err == io.EOF {
+					return
+				}
+				t.Fatalf("%s: %+v", test.source, err)
 			}
-			t.Fatalf("%s: %+v", test.source, err)
-		}
-		actual := fmt.Sprintf("%+v", value.Elem().Interface())
-		expect := fmt.Sprintf("%+v", test.value)
-		if actual != expect {
-			t.Fatalf("failed to test [%s], actual=[%s], expect=[%s]", test.source, actual, expect)
-		}
+			actual := fmt.Sprintf("%+v", value.Elem().Interface())
+			expect := fmt.Sprintf("%+v", test.value)
+			if actual != expect {
+				t.Fatalf("failed to test [%s], actual=[%s], expect=[%s]", test.source, actual, expect)
+			}
+		})
 	}
 }
 
@@ -2174,3 +2176,30 @@ func TestDecoder_TabCharacterAtRight(t *testing.T) {
 		t.Fatalf("failed to unmarshal %+v", v)
 	}
 }
+
+func TestDecoder_Canonical(t *testing.T) {
+	yml := `
+!!map {
+  ? !!str "explicit":!!str "entry",
+  ? !!str "implicit" : !!str "entry",
+  ? !!null "" : !!null "",
+}
+`
+	var v interface{}
+	if err := yaml.Unmarshal([]byte(yml), &v); err != nil {
+		t.Fatalf("%+v", err)
+	}
+	m, ok := v.(map[string]interface{})
+	if !ok {
+		t.Fatalf("failed to decode canonical yaml: %+v", v)
+	}
+	if m["explicit"] != "entry" {
+		t.Fatalf("failed to decode canonical yaml: %+v", m)
+	}
+	if m["implicit"] != "entry" {
+		t.Fatalf("failed to decode canonical yaml: %+v", m)
+	}
+	if m["null"] != nil {
+		t.Fatalf("failed to decode canonical yaml: %+v", m)
+	}
+}

parser/parser.go

@@ -1,6 +1,7 @@
 package parser
 
 import (
+	"fmt"
 	"io/ioutil"
 	"strings"
 
@@ -26,13 +27,13 @@ func (p *parser) parseMapping(ctx *context) (ast.Node, error) {
 			continue
 		}
 
-		value, err := p.parseToken(ctx, tk)
+		value, err := p.parseMappingValue(ctx)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to parse mapping value in mapping node")
 		}
 		mvnode, ok := value.(*ast.MappingValueNode)
 		if !ok {
-			return nil, errors.ErrSyntax("failed to parse flow mapping value node", value.GetToken())
+			return nil, errors.ErrSyntax("failed to parse flow mapping node", value.GetToken())
 		}
 		node.Values = append(node.Values, mvnode)
 		ctx.progress(1)
@@ -64,9 +65,36 @@ func (p *parser) parseSequence(ctx *context) (ast.Node, error) {
 }
 
 func (p *parser) parseTag(ctx *context) (ast.Node, error) {
-	node := &ast.TagNode{Start: ctx.currentToken()}
+	tagToken := ctx.currentToken()
+	node := &ast.TagNode{Start: tagToken}
 	ctx.progress(1) // skip tag token
-	value, err := p.parseToken(ctx, ctx.currentToken())
+	var (
+		value ast.Node
+		err   error
+	)
+	switch token.ReservedTagKeyword(tagToken.Value) {
+	case token.MappingTag,
+		token.OrderedMapTag:
+		value, err = p.parseMapping(ctx)
+	case token.IntegerTag,
+		token.FloatTag,
+		token.StringTag,
+		token.BinaryTag,
+		token.TimestampTag,
+		token.NullTag:
+		typ := ctx.currentToken().Type
+		if typ == token.LiteralType || typ == token.FoldedType {
+			value, err = p.parseLiteral(ctx)
+		} else {
+			value = p.parseScalarValue(ctx.currentToken())
+		}
+	case token.SequenceTag,
+		token.SetTag:
+		err = errors.ErrSyntax(fmt.Sprintf("sorry, currently not supported %s tag", tagToken.Value), tagToken)
+	default:
+		// custom tag
+		value, err = p.parseToken(ctx, ctx.currentToken())
+	}
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse tag value")
 	}
@@ -158,16 +186,13 @@ func (p *parser) validateMapValue(ctx *context, key, value ast.Node) error {
 }
 
 func (p *parser) parseMappingValue(ctx *context) (ast.Node, error) {
-	key := p.parseMapKey(ctx.currentToken())
-	if key == nil {
-		return nil, errors.ErrSyntax("unexpected mapping 'key'. key is undefined", ctx.currentToken())
+	key, err := p.parseMapKey(ctx)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse map key")
 	}
 	if err := p.validateMapKey(key.GetToken()); err != nil {
 		return nil, errors.Wrapf(err, "validate mapping key error")
 	}
-	if _, ok := key.(ast.ScalarNode); !ok {
-		return nil, errors.ErrSyntax("unexpected mapping 'key', key is not scalar value", key.GetToken())
-	}
 	ctx.progress(1)          // progress to mapping value token
 	tk := ctx.currentToken() // get mapping value token
 	ctx.progress(1)          // progress to value token
@@ -297,17 +322,18 @@ func (p *parser) parseAlias(ctx *context) (ast.Node, error) {
 	return alias, nil
 }
 
-func (p *parser) parseMapKey(tk *token.Token) ast.Node {
-	if node := p.parseStringValue(tk); node != nil {
-		return node
-	}
-	if tk.Type == token.MergeKeyType {
-		return ast.MergeKey(tk)
+func (p *parser) parseMapKey(ctx *context) (ast.Node, error) {
+	tk := ctx.currentToken()
+	if value := p.parseScalarValue(tk); value != nil {
+		return value, nil
 	}
-	if tk.Type == token.NullType {
-		return ast.Null(tk)
+	switch tk.Type {
+	case token.MergeKeyType:
+		return ast.MergeKey(tk), nil
+	case token.MappingKeyType:
+		return p.parseMappingKey(ctx)
 	}
-	return nil
+	return nil, errors.ErrSyntax("unexpected mapping key", tk)
 }
 
 func (p *parser) parseStringValue(tk *token.Token) ast.Node {
@@ -460,6 +486,17 @@ func (p *parser) parseComment(ctx *context) (ast.Node, error) {
 	return node, nil
 }
 
+func (p *parser) parseMappingKey(ctx *context) (ast.Node, error) {
+	node := ast.MappingKey(ctx.currentToken())
+	ctx.progress(1) // skip mapping key token
+	value, err := p.parseToken(ctx, ctx.currentToken())
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse map key")
+	}
+	node.Value = value
+	return node, nil
+}
+
 func (p *parser) parseToken(ctx *context, tk *token.Token) (ast.Node, error) {
 	if tk == nil {
 		return nil, nil
@@ -478,6 +515,8 @@ func (p *parser) parseToken(ctx *context, tk *token.Token) (ast.Node, error) {
 	switch tk.Type {
 	case token.CommentType:
 		return p.parseComment(ctx)
+	case token.MappingKeyType:
+		return p.parseMappingKey(ctx)
 	case token.DocumentHeaderType:
 		return p.parseDocument(ctx)
 	case token.MappingStartType: