From 2baea7ddc25592411af832a15d15c34f03a41381 Mon Sep 17 00:00:00 2001 From: Wondertan Date: Sun, 1 Oct 2023 13:58:29 +0200 Subject: [PATCH] introduce File interface and decouple ipldv2 tests from on disk file --- share/eds/file.go | 49 +++++++++++----------- share/eds/file_store.go | 2 +- share/eds/file_test.go | 16 ------- share/eds/ods_file.go | 74 +++++++++++++++++++++++++++++++++ share/ipldv2/axis_sample.go | 2 +- share/ipldv2/blockstore.go | 20 ++------- share/ipldv2/blockstore_test.go | 33 +++++---------- share/ipldv2/ipldv2_test.go | 38 ++++------------- 8 files changed, 122 insertions(+), 112 deletions(-) create mode 100644 share/eds/ods_file.go diff --git a/share/eds/file.go b/share/eds/file.go index 69d7a4bc37..f353b13002 100644 --- a/share/eds/file.go +++ b/share/eds/file.go @@ -14,6 +14,15 @@ import ( "github.com/celestiaorg/celestia-node/share" ) +type File interface { + io.Closer + Size() int + ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) + Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) + AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) + EDS() (*rsmt2d.ExtendedDataSquare, error) +} + type FileConfig struct { Version FileVersion Compression FileCompression @@ -23,13 +32,13 @@ type FileConfig struct { // TODO: Add codec } -// File +// LazyFile // * immutable // * versionable // TODO: // - Cache Rows and Cols // - Avoid storing constant shares, like padding -type File struct { +type LazyFile struct { path string hdr *Header fl fileBackend @@ -40,7 +49,7 @@ type fileBackend interface { io.Closer } -func OpenFile(path string) (*File, error) { +func OpenFile(path string) (*LazyFile, error) { f, err := mmap.Open(path) if err != nil { return nil, err @@ -52,14 +61,14 @@ func OpenFile(path string) (*File, error) { } // TODO(WWondertan): Validate header - return &File{ + return &LazyFile{ path: path, hdr: h, fl: f, }, nil } -func CreateFile(path string, eds *rsmt2d.ExtendedDataSquare, cfgs ...FileConfig) (*File, error) { +func CreateFile(path string, eds *rsmt2d.ExtendedDataSquare, cfgs ...FileConfig) (*LazyFile, error) { f, err := os.Create(path) if err != nil { return nil, err @@ -94,22 +103,26 @@ func CreateFile(path string, eds *rsmt2d.ExtendedDataSquare, cfgs ...FileConfig) } } - return &File{ + return &LazyFile{ path: path, fl: f, hdr: h, }, f.Sync() } -func (f *File) Close() error { +func (f *LazyFile) Size() int { + return f.hdr.SquareSize() +} + +func (f *LazyFile) Close() error { return f.fl.Close() } -func (f *File) Header() *Header { +func (f *LazyFile) Header() *Header { return f.hdr } -func (f *File) Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) { +func (f *LazyFile) Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) { shrLn := int(f.hdr.shareSize) sqrLn := int(f.hdr.squareSize) if f.Header().Config().Mode == ODSMode { @@ -156,7 +169,7 @@ func (f *File) Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) { return shrs, nil } -func (f *File) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) { +func (f *LazyFile) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) { // TODO(@Wondertan): this has to read directly from the file, avoiding recompute fullAxis, err := f.Axis(idx, axis) if err != nil { @@ -166,19 +179,7 @@ func (f *File) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) { return fullAxis[:len(fullAxis)/2], nil } -func (f *File) Share(idx int) (share.Share, error) { - // TODO: Check the cache first - shrLn := int64(f.hdr.shareSize) - - offset := int64(idx)*shrLn + HeaderSize - shr := make(share.Share, shrLn) - if _, err := f.fl.ReadAt(shr, offset); err != nil { - return nil, err - } - return shr, nil -} - -func (f *File) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) { +func (f *LazyFile) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) { // TODO: Cache the axis as well as computed tree sqrLn := int(f.hdr.squareSize) axsIdx, shrIdx := idx/sqrLn, idx%sqrLn @@ -207,7 +208,7 @@ func (f *File) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof return shrs[shrIdx], proof, nil } -func (f *File) EDS() (*rsmt2d.ExtendedDataSquare, error) { +func (f *LazyFile) EDS() (*rsmt2d.ExtendedDataSquare, error) { shrLn := int(f.hdr.shareSize) sqrLn := int(f.hdr.squareSize) if f.Header().Config().Mode == ODSMode { diff --git a/share/eds/file_store.go b/share/eds/file_store.go index efbf968fed..d580bdc43d 100644 --- a/share/eds/file_store.go +++ b/share/eds/file_store.go @@ -6,7 +6,7 @@ type FileStore struct { baspath string } -func (fs *FileStore) File(hash share.DataHash) (*File, error) { +func (fs *FileStore) File(hash share.DataHash) (File, error) { // TODO(@Wondertan): Caching return OpenFile(fs.baspath + "/" + hash.String()) } diff --git a/share/eds/file_test.go b/share/eds/file_test.go index 901158f030..53e1bad8c8 100644 --- a/share/eds/file_test.go +++ b/share/eds/file_test.go @@ -54,10 +54,6 @@ func TestFile(t *testing.T) { for _, axis := range axis { for i := 0; i < width*width; i++ { row, col := uint(i/width), uint(i%width) - shr, err := fl.Share(i) - require.NoError(t, err) - assert.EqualValues(t, eds.GetCell(row, col), shr) - shr, prf, err := fl.ShareWithProof(i, axis) require.NoError(t, err) assert.EqualValues(t, eds.GetCell(row, col), shr) @@ -84,15 +80,3 @@ func TestFile(t *testing.T) { err = fl.Close() require.NoError(t, err) } - -// TODO(@Wondertan): Should be a method on eds -func getAxis(idx int, axis rsmt2d.Axis, eds *rsmt2d.ExtendedDataSquare) [][]byte { - switch axis { - case rsmt2d.Row: - return eds.Row(uint(idx)) - case rsmt2d.Col: - return eds.Col(uint(idx)) - default: - panic("") - } -} diff --git a/share/eds/ods_file.go b/share/eds/ods_file.go new file mode 100644 index 0000000000..6930174cc1 --- /dev/null +++ b/share/eds/ods_file.go @@ -0,0 +1,74 @@ +package eds + +import ( + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" +) + +type MemFile struct { + Eds *rsmt2d.ExtendedDataSquare +} + +func (f *MemFile) Close() error { + return nil +} + +func (f *MemFile) Size() int { + return int(f.Eds.Width()) +} + +func (f *MemFile) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) { + sqrLn := f.Size() + axsIdx, shrIdx := idx/sqrLn, idx%sqrLn + if axis == rsmt2d.Col { + axsIdx, shrIdx = shrIdx, axsIdx + } + + shrs, err := f.Axis(axsIdx, axis) + if err != nil { + return nil, nmt.Proof{}, err + } + + // TODO(@Wondartan): this must access cached NMT on EDS instead of computing a new one + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(sqrLn/2), uint(axsIdx)) + for _, shr := range shrs { + err = tree.Push(shr) + if err != nil { + return nil, nmt.Proof{}, err + } + } + + proof, err := tree.ProveRange(shrIdx, shrIdx+1) + if err != nil { + return nil, nmt.Proof{}, err + } + + return shrs[shrIdx], proof, nil +} + +func (f *MemFile) Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) { + return getAxis(idx, axis, f.Eds), nil +} + +func (f *MemFile) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) { + return getAxis(idx, axis, f.Eds)[:f.Size()/2], nil +} + +func (f *MemFile) EDS() (*rsmt2d.ExtendedDataSquare, error) { + return f.Eds, nil +} + +// TODO(@Wondertan): Should be a method on eds +func getAxis(idx int, axis rsmt2d.Axis, eds *rsmt2d.ExtendedDataSquare) [][]byte { + switch axis { + case rsmt2d.Row: + return eds.Row(uint(idx)) + case rsmt2d.Col: + return eds.Col(uint(idx)) + default: + panic("unknown axis") + } +} diff --git a/share/ipldv2/axis_sample.go b/share/ipldv2/axis_sample.go index b9379278ce..75bed9b0c1 100644 --- a/share/ipldv2/axis_sample.go +++ b/share/ipldv2/axis_sample.go @@ -134,7 +134,7 @@ func (s *AxisSample) Validate() error { } s.AxisHalf = append(s.AxisHalf, parity...) - tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(len(s.AxisHalf)), uint(s.ID.Index)) + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(len(s.AxisHalf)/2), uint(s.ID.Index)) for _, shr := range s.AxisHalf { err := tree.Push(shr) if err != nil { diff --git a/share/ipldv2/blockstore.go b/share/ipldv2/blockstore.go index a6c0f4e46b..6ac55b358b 100644 --- a/share/ipldv2/blockstore.go +++ b/share/ipldv2/blockstore.go @@ -3,38 +3,26 @@ package ipldv2 import ( "context" "fmt" - "io" "github.com/ipfs/boxo/blockstore" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/celestiaorg/nmt" - "github.com/celestiaorg/rsmt2d" - "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds" ) -// edsFile is a mocking friendly local interface over eds.File. -// TODO(@Wondertan): Consider making an actual interface of eds pkg -type edsFile interface { - io.Closer - Size() int - ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) - AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) -} - // fileStore is a mocking friendly local interface over eds.FileStore // TODO(@Wondertan): Consider making an actual interface of eds pkg -type fileStore[F edsFile] interface { +type fileStore[F eds.File] interface { File(share.DataHash) (F, error) } -type Blockstore[F edsFile] struct { +type Blockstore[F eds.File] struct { fs fileStore[F] } -func NewBlockstore[F edsFile](fs fileStore[F]) blockstore.Blockstore { +func NewBlockstore[F eds.File](fs fileStore[F]) blockstore.Blockstore { return &Blockstore[F]{fs} } diff --git a/share/ipldv2/blockstore_test.go b/share/ipldv2/blockstore_test.go index 044378aa4e..aef1c7e5b6 100644 --- a/share/ipldv2/blockstore_test.go +++ b/share/ipldv2/blockstore_test.go @@ -4,10 +4,10 @@ import ( "context" "testing" + "github.com/ipfs/boxo/blockstore" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -15,16 +15,15 @@ import ( "github.com/celestiaorg/celestia-node/share/eds/edstest" ) -func TestBlockstoreGet(t *testing.T) { +// TODO(@Wondertan): Add axis sampling code + +func TestBlockstoreGetShareSample(t *testing.T) { ctx := context.Background() sqr := edstest.RandEDS(t, 4) root, err := share.NewRoot(sqr) require.NoError(t, err) - path := t.TempDir() + "/eds_file" - f, err := eds.CreateFile(path, sqr) - require.NoError(t, err) - b := NewBlockstore[*edsFileAndFS]((*edsFileAndFS)(f)) + b := edsBlockstore(sqr) axis := []rsmt2d.Axis{rsmt2d.Row, rsmt2d.Col} width := int(sqr.Width()) @@ -47,24 +46,12 @@ func TestBlockstoreGet(t *testing.T) { } } -type edsFileAndFS eds.File - -func (m *edsFileAndFS) File(share.DataHash) (*edsFileAndFS, error) { - return m, nil -} - -func (m *edsFileAndFS) Size() int { - return (*eds.File)(m).Header().SquareSize() -} - -func (m *edsFileAndFS) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) { - return (*eds.File)(m).ShareWithProof(idx, axis) -} +type edsFileAndFS eds.MemFile -func (m *edsFileAndFS) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) { - return (*eds.File)(m).AxisHalf(idx, axis) +func (m *edsFileAndFS) File(share.DataHash) (*eds.MemFile, error) { + return (*eds.MemFile)(m), nil } -func (m *edsFileAndFS) Close() error { - return nil +func edsBlockstore(sqr *rsmt2d.ExtendedDataSquare) blockstore.Blockstore { + return NewBlockstore[*eds.MemFile]((*edsFileAndFS)(&eds.MemFile{Eds: sqr})) } diff --git a/share/ipldv2/ipldv2_test.go b/share/ipldv2/ipldv2_test.go index d32fe39e61..e2fc8d95c8 100644 --- a/share/ipldv2/ipldv2_test.go +++ b/share/ipldv2/ipldv2_test.go @@ -20,7 +20,6 @@ import ( "github.com/celestiaorg/rsmt2d" - "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/edstest" ) @@ -31,13 +30,7 @@ func TestShareSampleRoundtripGetBlock(t *testing.T) { defer cancel() sqr := edstest.RandEDS(t, 8) - - path := t.TempDir() + "/eds_file" - f, err := eds.CreateFile(path, sqr) - require.NoError(t, err) - defer f.Close() - - b := NewBlockstore[*edsFileAndFS]((*edsFileAndFS)(f)) + b := edsBlockstore(sqr) client := remoteClient(ctx, t, b) axis := []rsmt2d.Axis{rsmt2d.Col, rsmt2d.Row} @@ -68,13 +61,7 @@ func TestShareSampleRoundtripGetBlocks(t *testing.T) { defer cancel() sqr := edstest.RandEDS(t, 8) // TODO(@Wondertan): does not work with more than 8 for some reasong - - path := t.TempDir() + "/eds_file" - f, err := eds.CreateFile(path, sqr) - require.NoError(t, err) - defer f.Close() - - b := NewBlockstore[*edsFileAndFS]((*edsFileAndFS)(f)) + b := edsBlockstore(sqr) client := remoteClient(ctx, t, b) set := cid.NewSet() @@ -93,7 +80,7 @@ func TestShareSampleRoundtripGetBlocks(t *testing.T) { } blks := client.GetBlocks(ctx, set.Keys()) - err = set.ForEach(func(c cid.Cid) error { + err := set.ForEach(func(c cid.Cid) error { select { case blk := <-blks: assert.True(t, set.Has(blk.Cid())) @@ -112,17 +99,11 @@ func TestShareSampleRoundtripGetBlocks(t *testing.T) { } func TestAxisSampleRoundtripGetBlock(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10000) defer cancel() sqr := edstest.RandEDS(t, 8) - - path := t.TempDir() + "/eds_file" - f, err := eds.CreateFile(path, sqr) - require.NoError(t, err) - defer f.Close() - - b := NewBlockstore[*edsFileAndFS]((*edsFileAndFS)(f)) + b := edsBlockstore(sqr) client := remoteClient(ctx, t, b) axis := []rsmt2d.Axis{rsmt2d.Col, rsmt2d.Row} @@ -153,12 +134,7 @@ func TestAxisSampleRoundtripGetBlocks(t *testing.T) { defer cancel() sqr := edstest.RandEDS(t, 16) - - path := t.TempDir() + "/eds_file" - f, err := eds.CreateFile(path, sqr) - require.NoError(t, err) - - b := NewBlockstore[*edsFileAndFS]((*edsFileAndFS)(f)) + b := edsBlockstore(sqr) client := remoteClient(ctx, t, b) set := cid.NewSet() @@ -177,7 +153,7 @@ func TestAxisSampleRoundtripGetBlocks(t *testing.T) { } blks := client.GetBlocks(ctx, set.Keys()) - err = set.ForEach(func(c cid.Cid) error { + err := set.ForEach(func(c cid.Cid) error { select { case blk := <-blks: assert.True(t, set.Has(blk.Cid()))