Skip to content

Commit

Permalink
Include unknown fields from decoding into output when re-encoding
Browse files Browse the repository at this point in the history 
When a message is decoded that has fields the current module
doesn't utilize, just append them to the end of the message
when re-encoding a message from this instance.

The conformance suite tests for this.
  • Loading branch information
@rwstauner
rwstauner committed Jan 7, 2025
1 parent 939e2be commit 26de9f8
Show file tree
Hide file tree
Showing 19 changed files with 1,943 additions and 332 deletions.
50 changes: 35 additions & 15 deletions lib/protoboeuf/codegen.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ def encode
type_signature(params: { buff: "String" }, returns: "String", newline: true) +
"def _encode(buff)\n" +
fields.map { |field| encode_subtype(field) }.compact.join("\n") +
"\nbuff\n end\n\n"
"buff << @_unknown_fields if @_unknown_fields\nbuff\n end\n\n"
end

def encode_subtype(field, value_expr = iv_name(field), tagged = true)
Expand Down Expand Up @@ -469,18 +469,24 @@ def encode_int64(field, value_expr, tagged)
val = #{value_expr}
if val != 0
#{encode_tag_and_length(field, tagged)}
while val != 0
byte = val & 0x7F
#{encode_varint}
end
RUBY
end

val >>= 7
# This drops the top bits,
# Otherwise, with a signed right shift,
# we get infinity one bits at the top
val &= (1 << 57) - 1
def encode_varint(dest = "buff")
<<~RUBY
while val != 0
byte = val & 0x7F
byte |= 0x80 if val != 0
buff << byte
end
val >>= 7
# This drops the top bits,
# Otherwise, with a signed right shift,
# we get infinity one bits at the top
val &= (1 << 57) - 1
byte |= 0x80 if val != 0
#{dest} << byte
end
RUBY
end
Expand Down Expand Up @@ -1103,27 +1109,41 @@ def decode_from(buff, index, len)
# unexpected, so discard it and continue.
if !found
wire_type = tag & 0x7
unknown_bytes = +"".b
val = tag
<%= encode_varint("unknown_bytes") %>
case wire_type
when <%= VARINT %>
i = 0
while true
newbyte = buff.getbyte(index)
index += 1
if newbyte.nil? || newbyte < 0x80
break
end
break if newbyte.nil?
unknown_bytes << newbyte
break if newbyte < 0x80
i += 1
break if i > 9
end
when <%= I64 %>
unknown_bytes << buff.byteslice(index, 8)
index += 8
when <%= LEN %>
<%= pull_bytes("", "") %>
value = <%= pull_varint %>
val = value
<%= encode_varint("unknown_bytes") %>
unknown_bytes << buff.byteslice(index, value)
index += value
when <%= I32 %>
unknown_bytes << buff.byteslice(index, 4)
index += 4
else
raise "unknown wire type \#{wire_type}"
end
(@_unknown_fields ||= +"".b) << unknown_bytes
return self if index >= len
<%= pull_tag %>
end
Expand Down
44 changes: 38 additions & 6 deletions lib/protoboeuf/protobuf/any.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,20 +99,38 @@ def decode_from(buff, index, len)
# unexpected, so discard it and continue.
if !found
wire_type = tag & 0x7

unknown_bytes = +"".b
val = tag
while val != 0
byte = val & 0x7F

val >>= 7
# This drops the top bits,
# Otherwise, with a signed right shift,
# we get infinity one bits at the top
val &= (1 << 57) - 1

byte |= 0x80 if val != 0
unknown_bytes << byte
end

case wire_type
when 0
i = 0
while true
newbyte = buff.getbyte(index)
index += 1
break if newbyte.nil? || newbyte < 0x80
break if newbyte.nil?
unknown_bytes << newbyte
break if newbyte < 0x80
i += 1
break if i > 9
end
when 1
unknown_bytes << buff.byteslice(index, 8)
index += 8
when 2
## PULL_BYTES
value =
if (byte0 = buff.getbyte(index)) < 0x80
index += 1
Expand Down Expand Up @@ -168,15 +186,29 @@ def decode_from(buff, index, len)
raise "integer decoding error"
end

buff.byteslice(index, value)
index += value
val = value
while val != 0
byte = val & 0x7F

## END PULL_BYTES
val >>= 7
# This drops the top bits,
# Otherwise, with a signed right shift,
# we get infinity one bits at the top
val &= (1 << 57) - 1

byte |= 0x80 if val != 0
unknown_bytes << byte
end

unknown_bytes << buff.byteslice(index, value)
index += value
when 5
unknown_bytes << buff.byteslice(index, 4)
index += 4
else
raise "unknown wire type #{wire_type}"
end
(@_unknown_fields ||= +"".b) << unknown_bytes
return self if index >= len
## PULL_UINT64
tag =
Expand Down Expand Up @@ -514,7 +546,7 @@ def _encode(buff)

buff.concat(val.b)
end

buff << @_unknown_fields if @_unknown_fields
buff
end

Expand Down
44 changes: 38 additions & 6 deletions lib/protoboeuf/protobuf/boolvalue.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,20 +90,38 @@ def decode_from(buff, index, len)
# unexpected, so discard it and continue.
if !found
wire_type = tag & 0x7

unknown_bytes = +"".b
val = tag
while val != 0
byte = val & 0x7F

val >>= 7
# This drops the top bits,
# Otherwise, with a signed right shift,
# we get infinity one bits at the top
val &= (1 << 57) - 1

byte |= 0x80 if val != 0
unknown_bytes << byte
end

case wire_type
when 0
i = 0
while true
newbyte = buff.getbyte(index)
index += 1
break if newbyte.nil? || newbyte < 0x80
break if newbyte.nil?
unknown_bytes << newbyte
break if newbyte < 0x80
i += 1
break if i > 9
end
when 1
unknown_bytes << buff.byteslice(index, 8)
index += 8
when 2
## PULL_BYTES
value =
if (byte0 = buff.getbyte(index)) < 0x80
index += 1
Expand Down Expand Up @@ -159,15 +177,29 @@ def decode_from(buff, index, len)
raise "integer decoding error"
end

buff.byteslice(index, value)
index += value
val = value
while val != 0
byte = val & 0x7F

## END PULL_BYTES
val >>= 7
# This drops the top bits,
# Otherwise, with a signed right shift,
# we get infinity one bits at the top
val &= (1 << 57) - 1

byte |= 0x80 if val != 0
unknown_bytes << byte
end

unknown_bytes << buff.byteslice(index, value)
index += value
when 5
unknown_bytes << buff.byteslice(index, 4)
index += 4
else
raise "unknown wire type #{wire_type}"
end
(@_unknown_fields ||= +"".b) << unknown_bytes
return self if index >= len
## PULL_UINT64
tag =
Expand Down Expand Up @@ -310,7 +342,7 @@ def _encode(buff)
else
raise "bool values should be true or false"
end

buff << @_unknown_fields if @_unknown_fields
buff
end

Expand Down
44 changes: 38 additions & 6 deletions lib/protoboeuf/protobuf/bytesvalue.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,20 +90,38 @@ def decode_from(buff, index, len)
# unexpected, so discard it and continue.
if !found
wire_type = tag & 0x7

unknown_bytes = +"".b
val = tag
while val != 0
byte = val & 0x7F

val >>= 7
# This drops the top bits,
# Otherwise, with a signed right shift,
# we get infinity one bits at the top
val &= (1 << 57) - 1

byte |= 0x80 if val != 0
unknown_bytes << byte
end

case wire_type
when 0
i = 0
while true
newbyte = buff.getbyte(index)
index += 1
break if newbyte.nil? || newbyte < 0x80
break if newbyte.nil?
unknown_bytes << newbyte
break if newbyte < 0x80
i += 1
break if i > 9
end
when 1
unknown_bytes << buff.byteslice(index, 8)
index += 8
when 2
## PULL_BYTES
value =
if (byte0 = buff.getbyte(index)) < 0x80
index += 1
Expand Down Expand Up @@ -159,15 +177,29 @@ def decode_from(buff, index, len)
raise "integer decoding error"
end

buff.byteslice(index, value)
index += value
val = value
while val != 0
byte = val & 0x7F

## END PULL_BYTES
val >>= 7
# This drops the top bits,
# Otherwise, with a signed right shift,
# we get infinity one bits at the top
val &= (1 << 57) - 1

byte |= 0x80 if val != 0
unknown_bytes << byte
end

unknown_bytes << buff.byteslice(index, value)
index += value
when 5
unknown_bytes << buff.byteslice(index, 4)
index += 4
else
raise "unknown wire type #{wire_type}"
end
(@_unknown_fields ||= +"".b) << unknown_bytes
return self if index >= len
## PULL_UINT64
tag =
Expand Down Expand Up @@ -369,7 +401,7 @@ def _encode(buff)

buff.concat(val.b)
end

buff << @_unknown_fields if @_unknown_fields
buff
end

Expand Down
Loading

0 comments on commit 26de9f8

Please sign in to comment.