Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix EBML Lacer/Unlacer #140

Merged
merged 7 commits into from
Feb 16, 2021
Merged

Fix EBML Lacer/Unlacer #140

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f9d1902
Fix EBML unlacer
at-wat Feb 15, 2021
7f5e4e2
Fix lacer
at-wat Feb 16, 2021
6665115
Fix tests
at-wat Feb 16, 2021
d84c025
Add test cases for VInt enc/dec
at-wat Feb 16, 2021
e367a1c
Add more test cases
at-wat Feb 16, 2021
3cfd8d7
Add test data
at-wat Feb 16, 2021
24f9e00
Add OutOfRange error and test
at-wat Feb 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion block.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ func UnmarshalBlock(r io.Reader, n int64) (*Block, error) {
var b Block
var err error
var nRead int
if b.TrackNumber, nRead, err = readVInt(r); err != nil {
if b.TrackNumber, nRead, err = readVUInt(r); err != nil {
return nil, err
}
n -= int64(nRead)
Expand Down
2 changes: 1 addition & 1 deletion block_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func TestUnmarshalBlock(t *testing.T) {
},
},
"EBMLLace": {
[]byte{0x82, 0x01, 0x23, 0x06, 0x02, 0x81, 0x82, 0x0A, 0x0B, 0x1B, 0x0C},
[]byte{0x82, 0x01, 0x23, 0x06, 0x02, 0x81, 0xC0, 0x0A, 0x0B, 0x1B, 0x0C},
Block{
0x02, 0x0123, false, false, LacingEBML, false,
[][]byte{{0x0A}, {0x0B, 0x1B}, {0x0C}},
Expand Down
2 changes: 1 addition & 1 deletion elementtable.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ func init() {

func initReverseLookupTable(revTb elementRevTable, tb elementTable) {
for k, v := range tb {
e, _, err := readVInt(bytes.NewBuffer(v.b))
e, _, err := readVUInt(bytes.NewBuffer(v.b))
if err != nil {
panic(err)
}
Expand Down
14 changes: 12 additions & 2 deletions lacer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,8 +117,18 @@ func (l *ebmlLacer) Write(b [][]byte) error {
return wrapErrorf(ErrTooManyFrames, "lacing %d frames", nFrames)
}
size := []byte{byte(nFrames - 1)}
for i := 0; i < nFrames-1; i++ {
n, err := encodeElementID(uint64(len(b[i])))

n, err := encodeElementID(uint64(len(b[0])))
if err != nil {
return err
}
size = append(size, n...)

frameSizePrev := int64(len(b[0]))
for i := 1; i < nFrames-1; i++ {
frameSize := int64(len(b[i]))
n, err := encodeVInt(frameSize - frameSizePrev)
frameSizePrev = frameSize
if err != nil {
return err
}
Expand Down
16 changes: 8 additions & 8 deletions lacer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,21 +117,21 @@ func TestLacer(t *testing.T) {
"EBML": {
newLacer: NewEBMLLacer,
frames: [][]byte{
bytes.Repeat([]byte{0xAA}, 256),
bytes.Repeat([]byte{0xCC}, 16),
bytes.Repeat([]byte{0x55}, 8),
bytes.Repeat([]byte{0xAA}, 800),
bytes.Repeat([]byte{0xCC}, 500),
bytes.Repeat([]byte{0x55}, 100),
},
b: append(
[]byte{
0x02,
0x41, 0x00, // 256 bytes
0x90, // 16 bytes
0x43, 0x20, // 800 bytes
0x5E, 0xD3, // 500 bytes
Comment on lines +127 to +128
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Example data from: https://matroska.org/technical/basics.html#ebml-lacing

},
bytes.Join(
[][]byte{
bytes.Repeat([]byte{0xAA}, 256),
bytes.Repeat([]byte{0xCC}, 16),
bytes.Repeat([]byte{0x55}, 8),
bytes.Repeat([]byte{0xAA}, 800),
bytes.Repeat([]byte{0xCC}, 500),
bytes.Repeat([]byte{0x55}, 100),
}, []byte{})...),
err: nil,
},
Expand Down
17 changes: 15 additions & 2 deletions unlacer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,11 +141,24 @@ func NewEBMLUnlacer(r io.Reader, n int64) (Unlacer, error) {
r: r,
size: make([]int, nFrame),
}
for i := 0; i < nFrame-1; i++ {
n64, nRead, err := readVInt(ul.r)
un64, nRead, err := readVUInt(ul.r)
if err != nil {
return nil, err
}
n64 := int64(un64)
n -= int64(nRead)
ul.size[0] = int(n64)
ul.size[nFrame-1] -= int(n64)

for i := 1; i < nFrame-1; i++ {
n64Diff, nRead, err := readVInt(ul.r)
n64 += int64(n64Diff)
if err != nil {
return nil, err
}
if n64 < 0 {
return nil, io.ErrUnexpectedEOF
}
n -= int64(nRead)
ul.size[i] = int(n64)
ul.size[nFrame-1] -= int(n64)
Expand Down
37 changes: 28 additions & 9 deletions unlacer_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,13 +111,13 @@ func TestUnlacer(t *testing.T) {
newUnlacer: NewEBMLUnlacer,
header: []byte{
0x02,
0x41, 0x00, // 256 bytes
0x90, // 16 bytes
0x43, 0x20, // 800 bytes
0x5E, 0xD3, // 500 bytes
},
frames: [][]byte{
bytes.Repeat([]byte{0xAA}, 256),
bytes.Repeat([]byte{0xCC}, 16),
bytes.Repeat([]byte{0x55}, 8),
bytes.Repeat([]byte{0xAA}, 800),
bytes.Repeat([]byte{0xCC}, 500),
bytes.Repeat([]byte{0x55}, 100),
},
err: nil,
},
Expand All @@ -136,12 +136,31 @@ func TestUnlacer(t *testing.T) {
frames: [][]byte{},
err: io.ErrUnexpectedEOF,
},
"EBMLInvalidSize2": {
newUnlacer: NewEBMLUnlacer,
header: []byte{
0x03,
0x81,
},
frames: [][]byte{},
err: io.ErrUnexpectedEOF,
},
"EBMLNegativeSize": {
newUnlacer: NewEBMLUnlacer,
header: []byte{
0x03,
0x81, // 1 byte
0x80, // -62 bytes
},
frames: [][]byte{},
err: io.ErrUnexpectedEOF,
},
"EBMLMissingFrame": {
newUnlacer: NewEBMLUnlacer,
header: []byte{
0x02,
0x82,
0x81,
0x82, // 2 bytes
0x9F, // 2 bytes
},
frames: [][]byte{{0x00, 0x01}},
err: io.ErrUnexpectedEOF,
Expand All @@ -150,8 +169,8 @@ func TestUnlacer(t *testing.T) {
newUnlacer: NewEBMLUnlacer,
header: []byte{
0x02,
0x82,
0x81,
0x82, // 2 bytes
0x9E, // 1 byte
},
frames: [][]byte{{0x00, 0x01}, {0x02}},
err: io.ErrUnexpectedEOF,
Expand Down
2 changes: 1 addition & 1 deletion unmarshal.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func readElement(r0 io.Reader, n int64, vo reflect.Value, depth int, pos uint64,

for {
var headerSize uint64
e, nb, err := readVInt(r)
e, nb, err := readVUInt(r)
headerSize += uint64(nb)
if err != nil {
if nb == 0 && err == io.ErrUnexpectedEOF {
Expand Down
2 changes: 1 addition & 1 deletion unmarshal_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -500,7 +500,7 @@ func TestUnmarshal_Error(t *testing.T) {
"Block": {0xA3, 0x85, 0x81, 0x00, 0x00, 0x00, 0x00},
"BlockXiph": {0xA3, 0x88, 0x81, 0x00, 0x00, 0x02, 0x01, 0x01, 0x00, 0x00},
"BlockFixed": {0xA3, 0x87, 0x81, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00},
"BlockEBML": {0xA3, 0x88, 0x81, 0x00, 0x00, 0x06, 0x01, 0x81, 0x00, 0x00},
"BlockEBML": {0xA3, 0x88, 0x98, 0x00, 0x00, 0x06, 0x01, 0x81, 0x00, 0x00},
}
for name, b := range TestBinaries {
t.Run(name, func(t *testing.T) {
Expand Down
64 changes: 62 additions & 2 deletions value.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@ var ErrInvalidType = errors.New("invalid type")
// ErrUnsupportedElementID means that a value is out of range of EBML encoding.
var ErrUnsupportedElementID = errors.New("unsupported Element ID")

// ErrOutOfRange means that a value is out of range of the data type.
var ErrOutOfRange = errors.New("out of range")

var perTypeReader = map[DataType]func(io.Reader, uint64) (interface{}, error){
DataTypeInt: readInt,
DataTypeUInt: readUInt,
Expand All @@ -51,13 +54,13 @@ var perTypeReader = map[DataType]func(io.Reader, uint64) (interface{}, error){
}

func readDataSize(r io.Reader) (uint64, int, error) {
v, n, err := readVInt(r)
v, n, err := readVUInt(r)
if v == (uint64(0xFFFFFFFFFFFFFFFF) >> uint(64-n*7)) {
return SizeUnknown, n, err
}
return v, n, err
}
func readVInt(r io.Reader) (uint64, int, error) {
func readVUInt(r io.Reader) (uint64, int, error) {
var bs [1]byte
bytesRead, err := io.ReadFull(r, bs[:])
switch err {
Expand Down Expand Up @@ -118,6 +121,32 @@ func readVInt(r io.Reader) (uint64, int, error) {
vc--
}
}
func readVInt(r io.Reader) (int64, int, error) {
u, n, err := readVUInt(r)
if err != nil {
return 0, n, err
}
v := int64(u)
switch n {
case 1:
v -= 0x3F
case 2:
v -= 0x1FFF
case 3:
v -= 0x0FFFFF
case 4:
v -= 0x07FFFFFF
case 5:
v -= 0x03FFFFFFFF
case 6:
v -= 0x01FFFFFFFFFF
case 7:
v -= 0x00FFFFFFFFFFFF
default:
v -= 0x007FFFFFFFFFFFFF
}
return v, n, nil
}
func readBinary(r io.Reader, n uint64) (interface{}, error) {
bs := make([]byte, n)

Expand Down Expand Up @@ -257,6 +286,37 @@ func encodeElementID(v uint64) ([]byte, error) {
}
return nil, ErrUnsupportedElementID
}
func encodeVInt(v int64) ([]byte, error) {
switch {
case -0x3F <= v && v <= 0x3F:
v += 0x3F
return encodeDataSize(uint64(v), 1), nil
case -0x1FFF <= v && v <= 0x1FFF:
v += 0x1FFF
return encodeDataSize(uint64(v), 2), nil
case -0xFFFFF <= v && v <= 0xFFFFF:
v += 0xFFFFF
return encodeDataSize(uint64(v), 3), nil
case -0x7FFFFFF <= v && v <= 0x7FFFFFF:
v += 0x7FFFFFF
return encodeDataSize(uint64(v), 4), nil
case -0x3FFFFFFFF <= v && v <= 0x3FFFFFFFF:
v += 0x3FFFFFFFF
return encodeDataSize(uint64(v), 5), nil
case -0x1FFFFFFFFFF <= v && v <= 0x1FFFFFFFFFF:
v += 0x1FFFFFFFFFF
return encodeDataSize(uint64(v), 6), nil
case -0xFFFFFFFFFFFF <= v && v <= 0xFFFFFFFFFFFF:
v += 0xFFFFFFFFFFFF
return encodeDataSize(uint64(v), 7), nil
case -0x7FFFFFFFFFFFFF <= v && v <= 0x7FFFFFFFFFFFFF:
v += 0x7FFFFFFFFFFFFF
return encodeDataSize(uint64(v), 8), nil
default:
return nil, ErrOutOfRange
}
}

func encodeBinary(i interface{}, n uint64) ([]byte, error) {
v, ok := i.([]byte)
if !ok {
Expand Down
67 changes: 60 additions & 7 deletions value_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,12 @@ func TestDataSize(t *testing.T) {

for n, c := range testCases {
t.Run("DecodeVInt "+n, func(t *testing.T) {
r, _, err := readVInt(bytes.NewBuffer(c.b))
r, _, err := readVUInt(bytes.NewBuffer(c.b))
if err != nil {
t.Fatalf("Failed to readVInt: '%v'", err)
t.Fatalf("Failed to readVUInt: '%v'", err)
}
if r != c.i {
t.Errorf("Expected readVInt result: %d, got: %d", c.i, r)
t.Errorf("Expected readVUInt result: %d, got: %d", c.i, r)
}
})
}
Expand All @@ -51,7 +51,7 @@ func TestDataSize(t *testing.T) {
t.Fatalf("Failed to readDataSize: '%v'", err)
}
if r != c.i {
t.Errorf("Expected readVInt result: %d, got: %d", c.i, r)
t.Errorf("Expected readVUInt result: %d, got: %d", c.i, r)
}
})
}
Expand Down Expand Up @@ -112,12 +112,12 @@ func TestElementID(t *testing.T) {

for n, c := range testCases {
t.Run("Decode "+n, func(t *testing.T) {
r, _, err := readVInt(bytes.NewBuffer(c.b))
r, _, err := readVUInt(bytes.NewBuffer(c.b))
if err != nil {
t.Fatalf("Failed to readVInt: '%v'", err)
t.Fatalf("Failed to readVUInt: '%v'", err)
}
if r != c.i {
t.Errorf("Expected readVInt result: %d, got: %d", c.i, r)
t.Errorf("Expected readVUInt result: %d, got: %d", c.i, r)
}
})
}
Expand All @@ -137,7 +137,53 @@ func TestElementID(t *testing.T) {
if err != ErrUnsupportedElementID {
t.Errorf("Expected error type result: %s, got: %s", ErrUnsupportedElementID, err)
}
}

func TestVInt(t *testing.T) {
testCases := map[string]struct {
b []byte
i int64
}{
"1 byte (lower bound)": {[]byte{0x80}, -0x3F},
"1 byte (upper bound)": {[]byte{0xFE}, 0x3F},
"2 bytes (lower bound)": {[]byte{0x40, 0x00}, -0x1FFF},
"2 bytes (upper bound)": {[]byte{0x7F, 0xFE}, 0x1FFF},
"3 bytes (lower bound)": {[]byte{0x20, 0x00, 0x00}, -0xFFFFF},
"3 bytes (upper bound)": {[]byte{0x3F, 0xFF, 0xFE}, 0xFFFFF},
"4 bytes (lower bound)": {[]byte{0x10, 0x00, 0x00, 0x00}, -0x7FFFFFF},
"4 bytes (upper bound)": {[]byte{0x1F, 0xFF, 0xFF, 0xFE}, 0x7FFFFFF},
"5 bytes (lower bound)": {[]byte{0x08, 0x00, 0x00, 0x00, 0x00}, -0x3FFFFFFFF},
"5 bytes (upper bound)": {[]byte{0x0F, 0xFF, 0xFF, 0xFF, 0xFE}, 0x3FFFFFFFF},
"6 bytes (lower bound)": {[]byte{0x04, 0x00, 0x00, 0x00, 0x00, 0x00}, -0x1FFFFFFFFFF},
"6 bytes (upper bound)": {[]byte{0x07, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}, 0x1FFFFFFFFFF},
"7 bytes (lower bound)": {[]byte{0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, -0xFFFFFFFFFFFF},
"7 bytes (upper bound)": {[]byte{0x03, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}, 0xFFFFFFFFFFFF},
"8 bytes (lower bound)": {[]byte{0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, -0x7FFFFFFFFFFFFF},
"8 bytes (upper bound)": {[]byte{0x01, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFE}, 0x7FFFFFFFFFFFFF},
}

for n, c := range testCases {
t.Run("Decode "+n, func(t *testing.T) {
r, _, err := readVInt(bytes.NewBuffer(c.b))
if err != nil {
t.Fatalf("Failed to readVUInt: '%v'", err)
}
if r != c.i {
t.Errorf("Expected readVUInt result: %d, got: %d", c.i, r)
}
})
}
for n, c := range testCases {
t.Run("Encode "+n, func(t *testing.T) {
b, err := encodeVInt(c.i)
if err != nil {
t.Fatalf("Failed to encodeVInt: '%v'", err)
}
if !bytes.Equal(b, c.b) {
t.Errorf("Expected encodeVInt result: %d, got: %d", c.b, b)
}
})
}
}

func TestValue(t *testing.T) {
Expand Down Expand Up @@ -308,6 +354,13 @@ func TestEncodeValue_WrongSize(t *testing.T) {
}
}

func TestEncodeValue_OutOfRange(t *testing.T) {
_, err := encodeVInt(1<<63 - 1)
if err != ErrOutOfRange {
t.Fatalf("Expected error: '%v', got: '%v'", ErrOutOfRange, err)
}
}

func TestReadValue_WrongSize(t *testing.T) {
testCases := map[string]struct {
t DataType
Expand Down