-
Notifications
You must be signed in to change notification settings - Fork 497
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #325 from OffchainLabs/cbrotli
use cbrotli
- Loading branch information
Showing
27 changed files
with
1,226 additions
and
117 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| //go:build !js | ||
| // +build !js | ||
|
|
||
| package arbcompress | ||
|
|
||
| /* | ||
| #cgo CFLAGS: -g -Wall -I${SRCDIR}/../target/include/ | ||
| #cgo LDFLAGS: ${SRCDIR}/../target/lib/libbrotlidec-static.a ${SRCDIR}/../target/lib/libbrotlienc-static.a ${SRCDIR}/../target/lib/libbrotlicommon-static.a -lm | ||
| #include "brotli/encode.h" | ||
| #include "brotli/decode.h" | ||
| */ | ||
| import "C" | ||
| import ( | ||
| "fmt" | ||
| ) | ||
|
|
||
| func Decompress(input []byte, maxSize int) ([]byte, error) { | ||
| outbuf := make([]byte, maxSize) | ||
| outsize := C.size_t(maxSize) | ||
| res := C.BrotliDecoderDecompress(C.size_t(len(input)), (*C.uint8_t)(&input[0]), &outsize, (*C.uint8_t)(&outbuf[0])) | ||
| if res != 1 { | ||
| return nil, fmt.Errorf("failed decompression: %d", res) | ||
| } | ||
| if int(outsize) > maxSize { | ||
| return nil, fmt.Errorf("result too large: %d", outsize) | ||
| } | ||
| return outbuf[:outsize], nil | ||
| } | ||
|
|
||
| func compressLevel(input []byte, level int) ([]byte, error) { | ||
| maxOutSize := compressedBufferSizeFor(len(input)) | ||
| outbuf := make([]byte, maxOutSize) | ||
| outSize := C.size_t(maxOutSize) | ||
| var inputPtr *C.uint8_t | ||
| if len(input) > 0 { | ||
| inputPtr = (*C.uint8_t)(&input[0]) | ||
| } | ||
| res := C.BrotliEncoderCompress(C.int(level), C.BROTLI_DEFAULT_WINDOW, C.BROTLI_MODE_GENERIC, | ||
| C.size_t(len(input)), inputPtr, &outSize, (*C.uint8_t)(&outbuf[0])) | ||
| if res != 1 { | ||
| return nil, fmt.Errorf("failed compression: %d", res) | ||
| } | ||
| return outbuf[:outSize], nil | ||
| } | ||
|
|
||
| func CompressWell(input []byte) ([]byte, error) { | ||
| return compressLevel(input, LEVEL_WELL) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package arbcompress | ||
|
|
||
| const LEVEL_FAST = 0 | ||
| const LEVEL_WELL = 11 | ||
| const WINDOW_SIZE = 22 // BROTLI_DEFAULT_WINDOW | ||
|
|
||
| func compressedBufferSizeFor(len int) int { | ||
| return len + (len>>10)*8 + 64 // actual limit is: len + (len >> 14) * 4 + 6 | ||
| } | ||
|
|
||
| func CompressFast(input []byte) ([]byte, error) { | ||
| return compressLevel(input, LEVEL_FAST) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| package arbcompress | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "testing" | ||
|
|
||
| "github.com/offchainlabs/nitro/util/testhelpers" | ||
| ) | ||
|
|
||
| func testDecompress(t *testing.T, compressed, decompressed []byte) { | ||
| res, err := Decompress(compressed, len(decompressed)*2+64) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| if !bytes.Equal(res, decompressed) { | ||
| t.Fatal("results differ ", res, " vs. ", decompressed) | ||
| } | ||
| } | ||
|
|
||
| func testCompressDecompress(t *testing.T, data []byte) { | ||
| compressedWell, err := CompressWell(data) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| testDecompress(t, compressedWell, data) | ||
|
|
||
| compressedFast, err := CompressFast(data) | ||
| if err != nil { | ||
| t.Fatal(err) | ||
| } | ||
| testDecompress(t, compressedFast, data) | ||
| } | ||
|
|
||
| func TestArbCompress(t *testing.T) { | ||
| asciiData := []byte("This is a long and repetitive string. Yadda yadda yadda yadda yadda. The quick brown fox jumped over the lazy dog.") | ||
| for i := 0; i < 8; i++ { | ||
| asciiData = append(asciiData, asciiData...) | ||
| } | ||
| testCompressDecompress(t, asciiData) | ||
|
|
||
| source := testhelpers.NewPseudoRandomDataSource(t, 0) | ||
| randData := source.GetData(2500) | ||
| testCompressDecompress(t, randData) | ||
|
|
||
| // test empty data: | ||
| testCompressDecompress(t, []byte{}) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| //go:build js | ||
| // +build js | ||
|
|
||
| package arbcompress | ||
|
|
||
| import ( | ||
| "fmt" | ||
| ) | ||
|
|
||
| func brotliCompress(inBuf []byte, outBuf []byte, level int, windowSize int) int64 | ||
|
|
||
| func brotliDecompress(inBuf []byte, outBuf []byte) int64 | ||
|
|
||
| func Decompress(input []byte, maxSize int) ([]byte, error) { | ||
| outBuf := make([]byte, maxSize) | ||
| outLen := brotliDecompress(input, outBuf) | ||
| if outLen < 0 { | ||
| return nil, fmt.Errorf("failed decompression") | ||
| } | ||
| return outBuf[:outLen], nil | ||
| } | ||
|
|
||
| func compressLevel(input []byte, level int) ([]byte, error) { | ||
| maxOutSize := compressedBufferSizeFor(len(input)) | ||
| outBuf := make([]byte, maxOutSize) | ||
| outLen := brotliCompress(input, outBuf, level, WINDOW_SIZE) | ||
| if outLen < 0 { | ||
| return nil, fmt.Errorf("failed compression") | ||
| } | ||
| return outBuf[:outLen], nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| // | ||
| // Copyright 2021, Offchain Labs, Inc. All rights reserved. | ||
| // | ||
|
|
||
| //go:build js | ||
| // +build js | ||
|
|
||
| #include "textflag.h" | ||
|
|
||
| TEXT ·brotliCompress(SB), NOSPLIT, $0 | ||
| CallImport | ||
| RET | ||
|
|
||
| TEXT ·brotliDecompress(SB), NOSPLIT, $0 | ||
| CallImport | ||
| RET |
Oops, something went wrong.