diff --git a/third_party/celestia-app/shares/compact_shares_test.go b/third_party/celestia-app/shares/compact_shares_test.go index 18c66a490..f2f5756f3 100644 --- a/third_party/celestia-app/shares/compact_shares_test.go +++ b/third_party/celestia-app/shares/compact_shares_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - coretypes "github.com/cometbft/cometbft/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -15,24 +14,6 @@ import ( "github.com/rollkit/rollkit/third_party/celestia-app/testfactory" ) -func SplitTxs(txs coretypes.Txs) (txShares []Share, err error) { - txWriter := NewCompactShareSplitter(appns.TxNamespace, appconsts.ShareVersionZero) - - for _, tx := range txs { - err = txWriter.WriteTx(tx) - if err != nil { - return nil, err - } - } - - txShares, _, err = txWriter.Export(0) - if err != nil { - return nil, err - } - - return txShares, nil -} - func TestCompactShareSplitter(t *testing.T) { // note that this test is mainly for debugging purposes, the main round trip // tests occur in TestMerge and Test_processCompactShares @@ -131,6 +112,29 @@ func Test_processCompactShares(t *testing.T) { } } +func TestAllSplit(t *testing.T) { + txs := testfactory.GenerateRandomlySizedTxs(1000, 150) + txShares, err := SplitTxs(txs) + require.NoError(t, err) + resTxs, err := ParseTxs(txShares) + require.NoError(t, err) + assert.Equal(t, resTxs, txs) +} + +func TestParseRandomOutOfContextShares(t *testing.T) { + txs := testfactory.GenerateRandomlySizedTxs(1000, 150) + txShares, err := SplitTxs(txs) + require.NoError(t, err) + + for i := 0; i < 1000; i++ { + start, length := testfactory.GetRandomSubSlice(len(txShares)) + randomRange := NewRange(start, start+length) + resTxs, err := ParseTxs(txShares[randomRange.Start:randomRange.End]) + require.NoError(t, err) + assert.True(t, testfactory.CheckSubArray(txs, resTxs)) + } +} + func TestCompactShareContainsInfoByte(t *testing.T) { css := NewCompactShareSplitter(appns.TxNamespace, appconsts.ShareVersionZero) txs := testfactory.GenerateRandomTxs(1, appconsts.ContinuationCompactShareContentSize/4) @@ -197,10 +201,6 @@ func Test_parseCompactSharesErrors(t *testing.T) { } testCases := []testCase{ - { - "share with start indicator false", - txShares[1:], // set the first share to the second share which has the start indicator set to false - }, { "share with unsupported share version", []Share{*shareWithUnsupportedShareVersion}, diff --git a/third_party/celestia-app/shares/parse_compact_shares.go b/third_party/celestia-app/shares/parse_compact_shares.go index 32f959012..815f96a06 100644 --- a/third_party/celestia-app/shares/parse_compact_shares.go +++ b/third_party/celestia-app/shares/parse_compact_shares.go @@ -1,6 +1,12 @@ package shares -import "errors" +import ( + "github.com/rollkit/rollkit/third_party/celestia-app/appconsts" +) + +func ParseCompactShares(shares []Share) (data [][]byte, err error) { + return parseCompactShares(shares, appconsts.SupportedShareVersions) +} // parseCompactShares returns data (transactions or intermediate state roots // based on the contents of rawShares and supportedShareVersions. If rawShares @@ -13,14 +19,6 @@ func parseCompactShares(shares []Share, supportedShareVersions []uint8) (data [] return nil, nil } - seqStart, err := shares[0].IsSequenceStart() - if err != nil { - return nil, err - } - if !seqStart { - return nil, errors.New("first share is not the start of a sequence") - } - err = validateShareVersions(shares, supportedShareVersions) if err != nil { return nil, err @@ -61,19 +59,32 @@ func parseRawData(rawData []byte) (units [][]byte, err error) { if err != nil { return nil, err } + // the rest of raw data is padding if unitLen == 0 { return units, nil } + // the rest of actual data contains only part of the next transaction so + // we stop parsing raw data + if unitLen > uint64(len(actualData)) { + return units, nil + } rawData = actualData[unitLen:] units = append(units, actualData[:unitLen]) } } -// extractRawData returns the raw data contained in the shares. The raw data does -// not contain the namespace ID, info byte, sequence length, or reserved bytes. +// extractRawData returns the raw data representing complete transactions +// contained in the shares. The raw data does not contain the namespace, info +// byte, sequence length, or reserved bytes. Starts reading raw data based on +// the reserved bytes in the first share. func extractRawData(shares []Share) (rawData []byte, err error) { for i := 0; i < len(shares); i++ { - raw, err := shares[i].RawData() + var raw []byte + if i == 0 { + raw, err = shares[i].RawDataUsingReserved() + } else { + raw, err = shares[i].RawData() + } if err != nil { return nil, err } diff --git a/third_party/celestia-app/shares/range.go b/third_party/celestia-app/shares/range.go new file mode 100644 index 000000000..490cdbe46 --- /dev/null +++ b/third_party/celestia-app/shares/range.go @@ -0,0 +1,26 @@ +package shares + +// Range is an end exclusive set of share indexes. +type Range struct { + // Start is the index of the first share occupied by this range. + Start int + // End is the next index after the last share occupied by this range. + End int +} + +func NewRange(start, end int) Range { + return Range{Start: start, End: end} +} + +func EmptyRange() Range { + return Range{Start: 0, End: 0} +} + +func (r Range) IsEmpty() bool { + return r.Start == 0 && r.End == 0 +} + +func (r *Range) Add(value int) { + r.Start += value + r.End += value +} diff --git a/third_party/celestia-app/shares/share_builder.go b/third_party/celestia-app/shares/share_builder.go index ac88a686b..16362f758 100644 --- a/third_party/celestia-app/shares/share_builder.go +++ b/third_party/celestia-app/shares/share_builder.go @@ -123,12 +123,12 @@ func (b *Builder) indexOfInfoBytes() int { return appconsts.NamespaceSize } -// MaybeWriteReservedBytes will be a no-op if the reserved bytes +// MaybeWriteReservedBytes will be a no-op for a compact share or if the reserved bytes // have already been populated. If the reserved bytes are empty, it will write // the location of the next unit of data to the reserved bytes. func (b *Builder) MaybeWriteReservedBytes() error { if !b.isCompactShare { - return errors.New("this is not a compact share") + return nil } empty, err := b.isEmptyReservedBytes() diff --git a/third_party/celestia-app/shares/shares.go b/third_party/celestia-app/shares/shares.go index c07d001b4..d3e9d6785 100644 --- a/third_party/celestia-app/shares/shares.go +++ b/third_party/celestia-app/shares/shares.go @@ -161,6 +161,29 @@ func (s *Share) ToBytes() []byte { return s.data } +// RawDataWithReserved returns the raw share data including the reserved bytes. The raw share data does not contain the namespace ID, info byte, or sequence length. +func (s *Share) RawDataWithReserved() (rawData []byte, err error) { + if len(s.data) < s.rawDataStartIndexWithReserved() { + return rawData, fmt.Errorf("share %s is too short to contain raw data", s) + } + + return s.data[s.rawDataStartIndexWithReserved():], nil +} + +func (s *Share) rawDataStartIndexWithReserved() int { + isStart, err := s.IsSequenceStart() + if err != nil { + panic(err) + } + + index := appconsts.NamespaceSize + appconsts.ShareInfoBytes + if isStart { + index += appconsts.SequenceLenBytes + } + + return index +} + // RawData returns the raw share data. The raw share data does not contain the // namespace ID, info byte, sequence length, or reserved bytes. func (s *Share) RawData() (rawData []byte, err error) { @@ -171,6 +194,51 @@ func (s *Share) RawData() (rawData []byte, err error) { return s.data[s.rawDataStartIndex():], nil } +// RawDataWithReserved returns the raw share data while taking reserved bytes into account. +func (s *Share) RawDataUsingReserved() (rawData []byte, err error) { + rawDataStartIndexUsingReserved, err := s.rawDataStartIndexUsingReserved() + if err != nil { + return nil, err + } + + // This means share is the last share and does not have any transaction beginning in it + if rawDataStartIndexUsingReserved == 0 { + return []byte{}, nil + } + if len(s.data) < rawDataStartIndexUsingReserved { + return rawData, fmt.Errorf("share %s is too short to contain raw data", s) + } + + return s.data[rawDataStartIndexUsingReserved:], nil +} + +// rawDataStartIndexUsingReserved returns the start index of raw data while accounting for +// reserved bytes, if it exists in the share. +func (s *Share) rawDataStartIndexUsingReserved() (int, error) { + isStart, err := s.IsSequenceStart() + if err != nil { + return 0, err + } + isCompact, err := s.IsCompactShare() + if err != nil { + return 0, err + } + + index := appconsts.NamespaceSize + appconsts.ShareInfoBytes + if isStart { + index += appconsts.SequenceLenBytes + } + + if isCompact { + reservedBytes, err := ParseReservedBytes(s.data[index : index+appconsts.CompactShareReservedBytes]) + if err != nil { + return 0, err + } + return int(reservedBytes), nil + } + return index, nil +} + func (s *Share) rawDataStartIndex() int { isStart, err := s.IsSequenceStart() if err != nil { @@ -180,17 +248,15 @@ func (s *Share) rawDataStartIndex() int { if err != nil { panic(err) } - if isStart && isCompact { - return appconsts.NamespaceSize + appconsts.ShareInfoBytes + appconsts.SequenceLenBytes + appconsts.CompactShareReservedBytes - } else if isStart && !isCompact { - return appconsts.NamespaceSize + appconsts.ShareInfoBytes + appconsts.SequenceLenBytes - } else if !isStart && isCompact { - return appconsts.NamespaceSize + appconsts.ShareInfoBytes + appconsts.CompactShareReservedBytes - } else if !isStart && !isCompact { - return appconsts.NamespaceSize + appconsts.ShareInfoBytes - } else { - panic(fmt.Sprintf("unable to determine the rawDataStartIndex for share %s", s.data)) + + index := appconsts.NamespaceSize + appconsts.ShareInfoBytes + if isStart { + index += appconsts.SequenceLenBytes + } + if isCompact { + index += appconsts.CompactShareReservedBytes } + return index } func ToBytes(shares []Share) (bytes [][]byte) { diff --git a/third_party/celestia-app/shares/split_compact_shares.go b/third_party/celestia-app/shares/split_compact_shares.go index 1d944e9b9..2d7182185 100644 --- a/third_party/celestia-app/shares/split_compact_shares.go +++ b/third_party/celestia-app/shares/split_compact_shares.go @@ -51,6 +51,25 @@ func NewCompactShareSplitter(ns appns.Namespace, shareVersion uint8) *CompactSha } } +// NewCompactShareSplitterWithIsCompactFalse returns a CompactShareSplitter using the provided +// namespace and shareVersion. Also, sets isCompact in the builder to false. +func NewCompactShareSplitterWithIsCompactFalse(ns appns.Namespace, shareVersion uint8) *CompactShareSplitter { + builder := NewBuilder(ns, shareVersion, true) + builder.isCompactShare = false + sb, err := builder.Init() + if err != nil { + panic(err) + } + + return &CompactShareSplitter{ + shares: []Share{}, + namespace: ns, + shareVersion: shareVersion, + shareRanges: map[coretypes.TxKey]ShareRange{}, + shareBuilder: sb, + } +} + // WriteTx adds the delimited data for the provided tx to the underlying compact // share splitter. func (css *CompactShareSplitter) WriteTx(tx coretypes.Tx) error { @@ -73,6 +92,36 @@ func (css *CompactShareSplitter) WriteTx(tx coretypes.Tx) error { return nil } +// write adds the delimited data to the underlying compact shares. +func (css *CompactShareSplitter) WriteWithNoReservedBytes(rawData []byte) error { + if css.done { + // remove the last element + if !css.shareBuilder.IsEmptyShare() { + css.shares = css.shares[:len(css.shares)-1] + } + css.done = false + } + + for { + rawDataLeftOver := css.shareBuilder.AddData(rawData) + if rawDataLeftOver == nil { + break + } + if err := css.stackPendingWithIsCompactFalse(); err != nil { + return err + } + + rawData = rawDataLeftOver + } + + if css.shareBuilder.AvailableBytes() == 0 { + if err := css.stackPendingWithIsCompactFalse(); err != nil { + return err + } + } + return nil +} + // write adds the delimited data to the underlying compact shares. func (css *CompactShareSplitter) write(rawData []byte) error { if css.done { @@ -120,6 +169,21 @@ func (css *CompactShareSplitter) stackPending() error { return err } +// stackPending will build & add the pending share to accumulated shares +func (css *CompactShareSplitter) stackPendingWithIsCompactFalse() error { + pendingShare, err := css.shareBuilder.Build() + if err != nil { + return err + } + css.shares = append(css.shares, *pendingShare) + + // Now we need to create a new builder + builder := NewBuilder(css.namespace, css.shareVersion, false) + builder.isCompactShare = false + css.shareBuilder, err = builder.Init() + return err +} + // Export finalizes and returns the underlying compact shares and a map of // shareRanges. All share ranges in the map of shareRanges will be offset (i.e. // incremented) by the shareRangeOffset provided. shareRangeOffset should be 0 diff --git a/third_party/celestia-app/shares/utils.go b/third_party/celestia-app/shares/utils.go index 16e6d1b06..6ce495a53 100644 --- a/third_party/celestia-app/shares/utils.go +++ b/third_party/celestia-app/shares/utils.go @@ -5,6 +5,9 @@ import ( "encoding/binary" coretypes "github.com/cometbft/cometbft/types" + + "github.com/rollkit/rollkit/third_party/celestia-app/appconsts" + appns "github.com/rollkit/rollkit/third_party/celestia-app/namespace" ) // DelimLen calculates the length of the delimiter for a given unit size @@ -96,3 +99,38 @@ func ParseDelimiter(input []byte) (inputWithoutLenDelimiter []byte, unitLen uint // return the input without the length delimiter return input[n:], dataLen, nil } + +// ParseTxs collects all of the transactions from the shares provided +func ParseTxs(shares []Share) (coretypes.Txs, error) { + // parse the shares + rawTxs, err := parseCompactShares(shares, appconsts.SupportedShareVersions) + if err != nil { + return nil, err + } + + // convert to the Tx type + txs := make(coretypes.Txs, len(rawTxs)) + for i := 0; i < len(txs); i++ { + txs[i] = coretypes.Tx(rawTxs[i]) + } + + return txs, nil +} + +func SplitTxs(txs coretypes.Txs) (txShares []Share, err error) { + txWriter := NewCompactShareSplitter(appns.TxNamespace, appconsts.ShareVersionZero) + + for _, tx := range txs { + err = txWriter.WriteTx(tx) + if err != nil { + return nil, err + } + } + + txShares, _, err = txWriter.Export(0) + if err != nil { + return nil, err + } + + return txShares, nil +} diff --git a/third_party/celestia-app/testfactory/txs.go b/third_party/celestia-app/testfactory/txs.go index 932276ec3..be25ff73a 100644 --- a/third_party/celestia-app/testfactory/txs.go +++ b/third_party/celestia-app/testfactory/txs.go @@ -1,6 +1,7 @@ package testfactory import ( + "bytes" mrand "math/rand" "github.com/cometbft/cometbft/types" @@ -30,3 +31,28 @@ func GenerateRandomTxs(count, size int) types.Txs { } return txs } + +// GetRandomSubSlice returns two integers representing a randomly sized range in the interval [0, size] +func GetRandomSubSlice(size int) (start int, length int) { + length = mrand.Intn(size + 1) //nolint:gosec + start = mrand.Intn(size - length + 1) //nolint:gosec + return start, length +} + +// CheckSubArray returns whether subTxList is a subarray of txList +func CheckSubArray(txList []types.Tx, subTxList []types.Tx) bool { + for i := 0; i <= len(txList)-len(subTxList); i++ { + j := 0 + for j = 0; j < len(subTxList); j++ { + tx := txList[i+j] + subTx := subTxList[j] + if !bytes.Equal([]byte(tx), []byte(subTx)) { + break + } + } + if j == len(subTxList) { + return true + } + } + return false +} diff --git a/types/tx.go b/types/tx.go index f6ecbe7dd..e778f3321 100644 --- a/types/tx.go +++ b/types/tx.go @@ -7,6 +7,9 @@ import ( "github.com/cometbft/cometbft/crypto/tmhash" cmbytes "github.com/cometbft/cometbft/libs/bytes" + "github.com/rollkit/rollkit/third_party/celestia-app/appconsts" + appns "github.com/rollkit/rollkit/third_party/celestia-app/namespace" + "github.com/rollkit/rollkit/third_party/celestia-app/shares" pb "github.com/rollkit/rollkit/types/pb/rollkit" ) @@ -54,7 +57,7 @@ func (txs Txs) ToTxsWithISRs(intermediateStateRoots IntermediateStateRoots) ([]p if len(intermediateStateRoots.RawRootsList) != expectedISRListLength { return nil, fmt.Errorf("invalid length of ISR list: %d, expected length: %d", len(intermediateStateRoots.RawRootsList), expectedISRListLength) } - txsWithISRs := make([]pb.TxWithISRs, 0, len(txs)) + txsWithISRs := make([]pb.TxWithISRs, len(txs)) for i, tx := range txs { txsWithISRs[i] = pb.TxWithISRs{ PreIsr: intermediateStateRoots.RawRootsList[i], @@ -64,3 +67,54 @@ func (txs Txs) ToTxsWithISRs(intermediateStateRoots IntermediateStateRoots) ([]p } return txsWithISRs, nil } + +func TxsWithISRsToShares(txsWithISRs []pb.TxWithISRs) (txShares []shares.Share, err error) { + byteSlices := make([][]byte, len(txsWithISRs)) + for i, txWithISR := range txsWithISRs { + byteSlices[i], err = txWithISR.Marshal() + if err != nil { + return nil, err + } + } + coreTxs := shares.TxsFromBytes(byteSlices) + txShares, err = shares.SplitTxs(coreTxs) + return txShares, err +} + +func SharesToPostableBytes(txShares []shares.Share) (postableData []byte, err error) { + for i := 0; i < len(txShares); i++ { + raw, err := txShares[i].RawDataWithReserved() + if err != nil { + return nil, err + } + postableData = append(postableData, raw...) + } + return postableData, nil +} + +func PostableBytesToShares(postableData []byte) (txShares []shares.Share, err error) { + css := shares.NewCompactShareSplitterWithIsCompactFalse(appns.TxNamespace, appconsts.ShareVersionZero) + err = css.WriteWithNoReservedBytes(postableData) + if err != nil { + return nil, err + } + shares, _, err := css.Export(0) + return shares, err +} + +func SharesToTxsWithISRs(txShares []shares.Share) (txsWithISRs []pb.TxWithISRs, err error) { + byteSlices, err := shares.ParseCompactShares(txShares) + if err != nil { + return nil, err + } + txsWithISRs = make([]pb.TxWithISRs, len(byteSlices)) + for i, byteSlice := range byteSlices { + var txWithISR pb.TxWithISRs + err = txWithISR.Unmarshal(byteSlice) + if err != nil { + return nil, err + } + txsWithISRs[i] = txWithISR + } + return txsWithISRs, nil +} diff --git a/types/tx_test.go b/types/tx_test.go new file mode 100644 index 000000000..5ad26d3f7 --- /dev/null +++ b/types/tx_test.go @@ -0,0 +1,113 @@ +package types + +import ( + "bytes" + "math/rand" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/rollkit/rollkit/types/pb/rollkit" +) + +func TestTxWithISRSerializationRoundtrip(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + txs := make(Txs, 1000) + ISRs := IntermediateStateRoots{ + RawRootsList: make([][]byte, len(txs)+1), + } + for i := 0; i < len(txs); i++ { + txs[i] = GetRandomTx() + ISRs.RawRootsList[i] = GetRandomBytes(32) + } + ISRs.RawRootsList[len(txs)] = GetRandomBytes(32) + + txsWithISRs, err := txs.ToTxsWithISRs(ISRs) + require.NoError(err) + require.NotEmpty(txsWithISRs) + + txShares, err := TxsWithISRsToShares(txsWithISRs) + require.NoError(err) + require.NotEmpty(txShares) + + txBytes, err := SharesToPostableBytes(txShares) + require.NoError(err) + require.NotEmpty(txBytes) + + newTxShares, err := PostableBytesToShares(txBytes) + require.NoError(err) + require.NotEmpty(newTxShares) + + // Note that txShares and newTxShares are not necessarily equal because newTxShares might + // contain zero padding at the end and thus sequence length can differ + newTxsWithISRs, err := SharesToTxsWithISRs(newTxShares) + require.NoError(err) + require.NotEmpty(txsWithISRs) + + assert.Equal(txsWithISRs, newTxsWithISRs) +} + +func TestTxWithISRSerializationOutOfContextRoundtrip(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + + numTxs := 1000 + txs := make(Txs, numTxs) + ISRs := IntermediateStateRoots{ + RawRootsList: make([][]byte, len(txs)+1), + } + for i := 0; i < len(txs); i++ { + txs[i] = GetRandomTx() + ISRs.RawRootsList[i] = GetRandomBytes(32) + } + ISRs.RawRootsList[len(txs)] = GetRandomBytes(32) + + txsWithISRs, err := txs.ToTxsWithISRs(ISRs) + require.NoError(err) + require.NotEmpty(txsWithISRs) + + txShares, err := TxsWithISRsToShares(txsWithISRs) + require.NoError(err) + require.NotEmpty(txShares) + + newTxsWithISRs, err := SharesToTxsWithISRs(txShares) + require.NoError(err) + require.NotEmpty(newTxsWithISRs) + + assert.Equal(txsWithISRs, newTxsWithISRs) + + for i := 0; i < 1000; i++ { + numShares := rand.Int() % len(txShares) //nolint: gosec + startShare := rand.Int() % (len(txShares) - numShares + 1) //nolint: gosec + newTxsWithISRs, err := SharesToTxsWithISRs(txShares[startShare : startShare+numShares]) + require.NoError(err) + assert.True(checkSubArray(txsWithISRs, newTxsWithISRs)) + } +} + +// Returns whether subTxList is a subarray of txList +func checkSubArray(txList []rollkit.TxWithISRs, subTxList []rollkit.TxWithISRs) (bool, error) { + for i := 0; i <= len(txList)-len(subTxList); i++ { + j := 0 + for j = 0; j < len(subTxList); j++ { + tx, err := txList[i+j].Marshal() + if err != nil { + return false, err + } + subTx, err := subTxList[j].Marshal() + if err != nil { + return false, err + } + if !bytes.Equal(tx, subTx) { + break + } + } + if j == len(subTxList) { + return true, nil + } + } + return false, nil +}