From 61eb65bd223bb60bf9df86fe547f111ee8b2319a Mon Sep 17 00:00:00 2001 From: Jean Date: Wed, 8 Nov 2023 13:55:08 +0100 Subject: [PATCH 1/7] link purb-db to libpurb --- README.md | 2 +- go.mod | 36 +++++++++++++++++++- go.sum | 34 ++++++++++-------- store/kv/blob.go | 89 ++++++++++++++++++++++++++++++++++++++++++++++++ store/kv/db.go | 25 ++++++++++---- store/kv/purb.go | 1 - 6 files changed, 164 insertions(+), 23 deletions(-) create mode 100644 store/kv/blob.go delete mode 100644 store/kv/purb.go diff --git a/README.md b/README.md index c7feb4c..91bf3ad 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,2 @@ -# dela-purb +# purb-db A {key,value} storage that is PURBified before saving it to disk. diff --git a/go.mod b/go.mod index 7ccfc9a..2cc2f23 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,37 @@ -module dela-purb +module purb-db go 1.21 + +require ( + github.com/stretchr/testify v1.8.4 + go.dedis.ch/dela v0.0.0-20231011144949-4677467c030c + go.dedis.ch/kyber/v3 v3.1.1-0.20231024084410-31ea167adbbb + go.dedis.ch/libpurb v0.0.0-20231107160354-ffba29c17caf + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.5.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.10.0 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/rs/zerolog v1.31.0 // indirect + github.com/russross/blackfriday/v2 v2.0.1 // indirect + github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect + github.com/urfave/cli/v2 v2.2.0 // indirect + go.dedis.ch/fixbuf v1.0.3 // indirect + go.etcd.io/bbolt v1.3.5 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/sys v0.13.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum index abed086..6bb50ff 100644 --- a/go.sum +++ b/go.sum @@ -10,7 +10,7 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -45,10 +45,11 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -82,9 +83,9 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= -github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= @@ -102,12 +103,18 @@ github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= go.dedis.ch/dela v0.0.0-20231011144949-4677467c030c h1:3qMO3ewks1QptS5GJyz59HYprgBEVUjk+9BWI6vEsRc= go.dedis.ch/dela v0.0.0-20231011144949-4677467c030c/go.mod h1:Oh/WK8JMO0POQg7nR3u436u+HwsZwPqPzDWAasgmFAU= +go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= +go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= +go.dedis.ch/kyber/v3 v3.1.1-0.20231024084410-31ea167adbbb h1:G7GmitWgVMlkBaSzHJJo05bydYhLFCSPzKwCHTwf3NM= +go.dedis.ch/kyber/v3 v3.1.1-0.20231024084410-31ea167adbbb/go.mod h1:nL0a5f3E//lnOCKumtzMmb8qzykxOVsihqp8C1Eb5rc= +go.dedis.ch/libpurb v0.0.0-20231107160354-ffba29c17caf h1:kNjCdcUbtc4+ANLZWApokLVzkHuW1NPxb4fNpGzFjBY= +go.dedis.ch/libpurb v0.0.0-20231107160354-ffba29c17caf/go.mod h1:XCi40g75txGSLusqJOanJytpkxiQLJzRb9RRV9UN2p4= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -121,10 +128,9 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/store/kv/blob.go b/store/kv/blob.go new file mode 100644 index 0000000..dd03158 --- /dev/null +++ b/store/kv/blob.go @@ -0,0 +1,89 @@ +package kv + +import ( + "go.dedis.ch/kyber/v3/group/curve25519" + "go.dedis.ch/kyber/v3/util/key" + "go.dedis.ch/libpurb/libpurb" + "golang.org/x/xerrors" + + "go.dedis.ch/kyber/v3/util/random" +) + +type Blob struct { + purb *libpurb.Purb +} + +// NewBlob creates a new blob +func NewBlob() *Blob { + suitesInfo := getSuiteInfo() + simplified := true + + p := libpurb.NewPurb( + suitesInfo, + simplified, + random.New(), + ) + + p.Recipients = createRecipients(1) + + return &Blob{ + purb: p, + } +} + +// Encode encodes a slice of bytes into a blob +func (b *Blob) Encode(data []byte) ([]byte, error) { + err := b.purb.Encode(data) + blob := b.purb.ToBytes() + + return blob, err +} + +// Decode decodes a blob into a slice of bytes +func (b *Blob) Decode(blob []byte) ([]byte, error) { + success, decrypted, err := b.purb.Decode(blob) + + if !success { + xerrors.Errorf("Failed to decrypt blob", err) + } + + return decrypted, err +} + +// --------------------------------------------------------------------------- +// helper functions + +// see example in libpurb +func getSuiteInfo() libpurb.SuiteInfoMap { + info := make(libpurb.SuiteInfoMap) + cornerstoneLength := 32 // defined by Curve 25519 + entryPointLength := 16 + 4 + 4 + 16 // 16-byte symmetric key + 2 * 4-byte offset positions + 16-byte authentication tag + info[curve25519.NewBlakeSHA256Curve25519(true).String()] = &libpurb.SuiteInfo{ + AllowedPositions: []int{ + 12 + 0*cornerstoneLength, + 12 + 1*cornerstoneLength, + 12 + 3*cornerstoneLength, + 12 + 4*cornerstoneLength, + }, + CornerstoneLength: cornerstoneLength, EntryPointLength: entryPointLength, + } + return info +} + +// see example in libpurb +func createRecipients(n int) []libpurb.Recipient { + decs := make([]libpurb.Recipient, 0) + suites := []libpurb.Suite{curve25519.NewBlakeSHA256Curve25519(true)} + for _, suite := range suites { + for i := 0; i < n; i++ { + pair := key.NewKeyPair(suite) + decs = append(decs, libpurb.Recipient{ + SuiteName: suite.String(), + Suite: suite, + PublicKey: pair.Public, + PrivateKey: pair.Private, + }) + } + } + return decs +} diff --git a/store/kv/db.go b/store/kv/db.go index f51164d..67d2368 100644 --- a/store/kv/db.go +++ b/store/kv/db.go @@ -21,6 +21,7 @@ type purbDB struct { dbFilePath string db db purbIsOn bool + blob *Blob } // NewDB opens a new database to the given file. @@ -40,21 +41,30 @@ func NewDB(path string, purbIsOn bool) (DB, error) { return nil, xerrors.Errorf("failed to read DB file: %v", err) } - dp := &purbDB{ + p := &purbDB{ dbFilePath: path, db: make(db), purbIsOn: purbIsOn, + blob: NewBlob(), } - dp.Lock() // unlocked in Close() + p.Lock() // unlocked in Close() buffer := bytes.NewBuffer(data) - err = dp.deserialize(buffer) + if purbIsOn { + decrypted, err := p.blob.Decode(data) + if err != nil { + return nil, err + } + buffer.Write(decrypted) + } + + err = p.deserialize(buffer) if fmt.Sprint(err) != "EOF" { return nil, xerrors.Errorf("failed to initialize new DB file: %v", err) } - return dp, nil + return p, nil } // View implements kv.DB. It executes the read-only transaction in the context @@ -74,7 +84,6 @@ func (p *purbDB) View(fn func(ReadableTx) error) error { // Update implements kv.DB. It executes the writable transaction in the context // of the database. func (p *purbDB) Update(fn func(WritableTx) error) error { - tx := &dpTx{db: p.db} err := fn(tx) @@ -122,7 +131,11 @@ func (p *purbDB) savePurbified() error { } if p.purbIsOn { - panic("Not implemented") + blob, err := p.blob.Encode(data.Bytes()) + if err != nil { + return xerrors.Errorf("failed to purbify DB file: %v", err) + } + data.Write(blob) } err = os.WriteFile(p.dbFilePath, data.Bytes(), 0755) diff --git a/store/kv/purb.go b/store/kv/purb.go deleted file mode 100644 index b81f834..0000000 --- a/store/kv/purb.go +++ /dev/null @@ -1 +0,0 @@ -package kv From ed47afeb3f92657d1bc1403eaaea27c402202467 Mon Sep 17 00:00:00 2001 From: Jean Date: Wed, 8 Nov 2023 14:09:25 +0100 Subject: [PATCH 2/7] improve git flow --- .github/workflows/go_lint.yml | 19 +++++++++++----- .github/workflows/go_test.yml | 42 +++++++++-------------------------- 2 files changed, 25 insertions(+), 36 deletions(-) diff --git a/.github/workflows/go_lint.yml b/.github/workflows/go_lint.yml index bfa0855..35deb09 100644 --- a/.github/workflows/go_lint.yml +++ b/.github/workflows/go_lint.yml @@ -2,7 +2,7 @@ name: Go lint on: push: - branches: [ master ] + branches: [ main ] pull_request: types: - opened @@ -14,14 +14,23 @@ jobs: lint: runs-on: ubuntu-latest steps: - - name: Set up Go ^1.19 - uses: actions/setup-go@v3 + - name: Set up Go 1.21 + uses: actions/setup-go@v4 with: - go-version: ^1.19 + go-version: "=1.21" - - name: Check out code into the Go module directory + - name: Checkout uses: actions/checkout@v3 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of Sonar analysis + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + # documentation: https://github.com/golangci/golangci-lint-action#readme + version: latest + install-mode: "binary" + - name: Lint run: make lint diff --git a/.github/workflows/go_test.yml b/.github/workflows/go_test.yml index 5c46910..6793cff 100644 --- a/.github/workflows/go_test.yml +++ b/.github/workflows/go_test.yml @@ -2,7 +2,7 @@ name: Go test on: push: - branches: [ master ] + branches: [ main ] pull_request: types: - opened @@ -10,32 +10,28 @@ on: - reopened jobs: - test: strategy: matrix: - platform: [ubuntu-latest, macos-latest, windows-latest] + platform: [ubuntu-latest] #, macos-latest, windows-latest] runs-on: ${{matrix.platform}} env: LLVL: trace steps: - - name: Set up Go 1.19 - uses: actions/setup-go@v3 + - name: Checkout + uses: actions/checkout@v3 with: - go-version: "=1.19" + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of Sonar analysis + + - name: Set up Go 1.21 + uses: actions/setup-go@v4 + with: + go-version: "=1.21" - - name: Check out code into the Go module directory - uses: actions/checkout@v3 - - name: Test without coverage - env: - CRY_LVL: "warn" - if: matrix.platform == 'macos-latest' || matrix.platform == 'windows-latest' run: make test - name: Test with coverage - env: - CRY_LVL: "warn" if: matrix.platform == 'ubuntu-latest' run: make coverage @@ -45,25 +41,9 @@ jobs: with: args: > -Dsonar.organization=dedis - -Dsonar.projectKey=dedis_dela + -Dsonar.projectKey=dedis_purb-db -Dsonar.go.tests.reportPaths=report.json -Dsonar.go.coverage.reportPaths=profile.cov env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - - - name: Send coverage - if: matrix.platform == 'ubuntu-latest' - uses: shogo82148/actions-goveralls@v1 - with: - path-to-profile: profile.cov - parallel: true - - # notifies that all test jobs are finished. - finish: - needs: test - runs-on: ubuntu-latest - steps: - - uses: shogo82148/actions-goveralls@v1 - with: - parallel-finished: true \ No newline at end of file From 74ddfe0f43b3568685305766e2a1de22b3027469 Mon Sep 17 00:00:00 2001 From: Jean Date: Wed, 8 Nov 2023 14:09:35 +0100 Subject: [PATCH 3/7] improve git flow --- Makefile | 20 +++++++++----------- go.mod | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 0e0199b..f961cd9 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all tidy generate lint vet test coverage pushdoc +.PHONY: all tidy lint vet test coverage # Default "make" target to check locally that everything is ok, BEFORE pushing remotely all: lint vet test @@ -9,19 +9,17 @@ tidy: # Some packages are excluded from staticcheck due to deprecated warnings: #208. lint: tidy - # Coding style static check. - @go install honnef.co/go/tools/cmd/staticcheck@latest - staticcheck `go list ./... | grep -Ev "(go\.dedis\.ch/dela/internal/testing|go\.dedis\.ch/dela/mino/minogrpc/ptypes)"` + golangci-lint run vet: tidy - @echo "⚠️ Warning: the following only works with go >= 1.14" && \ - go install go.dedis.ch/dela/internal/mcheck && \ - go vet -vettool=`go env GOPATH`/bin/mcheck -commentLen -ifInit ./... + go vet go.dedis.ch/purb-db/... -# test runs all tests in DELA without coverage test: tidy - go test ./... + # Test without coverage + LLVL="" + go test go.dedis.ch/purb-db/... -# test runs all tests in DELA and generate a coverage output (to be used by sonarcloud) coverage: tidy - go test -json -covermode=count -coverprofile=profile.cov ./... | tee report.json + # Test and generate a coverage output usable by sonarcloud + LLVL="" + go test -json -covermode=count -coverpkg=purb-db/... -coverprofile=profile.cov go.dedis.ch/purb-db/... | tee report.json diff --git a/go.mod b/go.mod index 2cc2f23..c8eb99b 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module purb-db +module go.dedis.ch/purb-db go 1.21 From 84e30b92078db0b82c3a55ab6376e9fff0fcb74b Mon Sep 17 00:00:00 2001 From: Jean Date: Wed, 8 Nov 2023 14:55:42 +0100 Subject: [PATCH 4/7] fix lint warnings --- store/kv/blob.go | 2 +- store/kv/db.go | 9 +++++---- store/kv/db_test.go | 3 ++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/store/kv/blob.go b/store/kv/blob.go index dd03158..78ee15b 100644 --- a/store/kv/blob.go +++ b/store/kv/blob.go @@ -44,7 +44,7 @@ func (b *Blob) Decode(blob []byte) ([]byte, error) { success, decrypted, err := b.purb.Decode(blob) if !success { - xerrors.Errorf("Failed to decrypt blob", err) + err = xerrors.Errorf("Failed to decrypt blob: %v", err) } return decrypted, err diff --git a/store/kv/db.go b/store/kv/db.go index 67d2368..2e7df8c 100644 --- a/store/kv/db.go +++ b/store/kv/db.go @@ -27,11 +27,10 @@ type purbDB struct { // NewDB opens a new database to the given file. func NewDB(path string, purbIsOn bool) (DB, error) { f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0755) - defer f.Close() - if err != nil { return nil, xerrors.Errorf("failed to open DB file: %v", err) } + defer f.Close() stats, _ := f.Stat() s := stats.Size() @@ -87,12 +86,14 @@ func (p *purbDB) Update(fn func(WritableTx) error) error { tx := &dpTx{db: p.db} err := fn(tx) - if err != nil { return err } - p.savePurbified() + err = p.savePurbified() + if err != nil { + return err + } tx.onCommit() diff --git a/store/kv/db_test.go b/store/kv/db_test.go index 189a4cf..52d4871 100644 --- a/store/kv/db_test.go +++ b/store/kv/db_test.go @@ -164,12 +164,13 @@ func TestPurbBucket_Scan(t *testing.T) { require.NoError(t, b.Set([]byte{0}, []byte{0})) var i byte = 0 - b.Scan(nil, func(k, v []byte) error { + err = b.Scan(nil, func(k, v []byte) error { require.Equal(t, []byte{i}, k) require.Equal(t, []byte{i}, v) i += 7 return nil }) + require.NoError(t, err) require.Equal(t, byte(14), i) err = b.Scan([]byte{1}, func(k, v []byte) error { From e8ed1e9fcd18c99b56167cf78a1e952e010327a1 Mon Sep 17 00:00:00 2001 From: Jean Date: Wed, 8 Nov 2023 17:29:02 +0100 Subject: [PATCH 5/7] fix broken DB tests --- go.mod | 1 + go.sum | 5 ++++- store/kv/bucket.go | 31 +++++++++++++++++++++++-------- store/kv/db.go | 11 +++++++++-- store/kv/db_test.go | 4 ++-- store/kv/kv.go | 2 +- store/kv/transaction.go | 19 ++++++++++++++++--- 7 files changed, 56 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index c8eb99b..857f474 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( go.dedis.ch/dela v0.0.0-20231011144949-4677467c030c go.dedis.ch/kyber/v3 v3.1.1-0.20231024084410-31ea167adbbb go.dedis.ch/libpurb v0.0.0-20231107160354-ffba29c17caf + golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 ) diff --git a/go.sum b/go.sum index 6bb50ff..61f85ad 100644 --- a/go.sum +++ b/go.sum @@ -33,8 +33,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -115,6 +116,8 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/store/kv/bucket.go b/store/kv/bucket.go index c7e8461..d179de2 100644 --- a/store/kv/bucket.go +++ b/store/kv/bucket.go @@ -1,22 +1,31 @@ package kv import ( + "slices" "strings" + "golang.org/x/exp/maps" "golang.org/x/xerrors" ) type kv map[string][]byte +type kOrder []string // dpBucket implements kv.Bucket type dpBucket struct { - kv kv + Kv kv + idx kOrder +} + +func (b *dpBucket) updateIndex() { + b.idx = maps.Keys(b.Kv) + slices.Sort(b.idx) } // Get implements kv.Bucket. It returns the value associated to the key, or nil // if it does not exist. func (b *dpBucket) Get(key []byte) []byte { - v, found := b.kv[string(key)] + v, found := b.Kv[string(key)] if found { return v } @@ -25,13 +34,19 @@ func (b *dpBucket) Get(key []byte) []byte { // Set implements kv.Bucket. It sets the provided key to the value. func (t *dpBucket) Set(key, value []byte) error { - t.kv[string(key)] = value + if _, found := t.Kv[string(key)]; !found { + t.idx = append(t.idx, string(key)) + slices.Sort(t.idx) + } + + t.Kv[string(key)] = value + return nil } // Delete implements kv.Bucket. It deletes the key from the bucket. func (b *dpBucket) Delete(key []byte) error { - delete(b.kv, string(key)) + delete(b.Kv, string(key)) return nil } @@ -39,8 +54,8 @@ func (b *dpBucket) Delete(key []byte) error { // unspecified order. If the callback returns an error, the iteration is stopped // and the error returned to the caller. func (b *dpBucket) ForEach(fn func(k, v []byte) error) error { - for k, v := range b.kv { - err := fn([]byte(k), v) + for _, k := range b.idx { + err := fn([]byte(k), b.Kv[k]) if err != nil { return err } @@ -52,11 +67,11 @@ func (b *dpBucket) ForEach(fn func(k, v []byte) error) error { // sorted order. If the callback returns an error, the iteration is stopped and // the error returned to the caller. func (b *dpBucket) Scan(prefix []byte, fn func(k, v []byte) error) error { - for k, v := range b.kv { + for _, k := range b.idx { if !strings.HasPrefix(k, string(prefix)) { continue } - err := fn([]byte(k), v) + err := fn([]byte(k), b.Kv[k]) if err != nil { return xerrors.Errorf("failed to scan bucket: %v", err) } diff --git a/store/kv/db.go b/store/kv/db.go index 2e7df8c..4549f70 100644 --- a/store/kv/db.go +++ b/store/kv/db.go @@ -11,7 +11,7 @@ import ( ) // bucket // key // value -type db map[string]map[string][]byte +type db map[string]*dpBucket // DB is the DELA/PURB implementation of the KV database. // @@ -95,7 +95,9 @@ func (p *purbDB) Update(fn func(WritableTx) error) error { return err } - tx.onCommit() + if tx.onCommit != nil { + tx.onCommit() + } return nil } @@ -122,6 +124,11 @@ func (p *purbDB) deserialize(input *bytes.Buffer) error { decoder := gob.NewDecoder(input) err := decoder.Decode(&p.db) + + for _, x := range p.db { + x.updateIndex() + } + return err } diff --git a/store/kv/db_test.go b/store/kv/db_test.go index 52d4871..15b0013 100644 --- a/store/kv/db_test.go +++ b/store/kv/db_test.go @@ -10,7 +10,7 @@ import ( "golang.org/x/xerrors" ) -const delaTestDir = "dela-core-kv" +const delaTestDir = "dela-core-Kv" func TestPurbDB_OpenClose(t *testing.T) { dir, err := os.MkdirTemp(os.TempDir(), delaTestDir) @@ -181,7 +181,7 @@ func TestPurbBucket_Scan(t *testing.T) { err = b.Scan([]byte{}, func(k, v []byte) error { return xerrors.New("callback error") }) - require.EqualError(t, err, "callback error") + require.ErrorContains(t, err, "callback error") return nil }) diff --git a/store/kv/kv.go b/store/kv/kv.go index f4d1cbd..5df7db8 100644 --- a/store/kv/kv.go +++ b/store/kv/kv.go @@ -1,4 +1,4 @@ -// Package kv defines the abstraction for a key/value database. +// Package Kv defines the abstraction for a key/value database. // // The package also implements a default database implementation that is using // bbolt as the engine (https://github.com/etcd-io/bbolt). diff --git a/store/kv/transaction.go b/store/kv/transaction.go index ceadf58..be943fc 100644 --- a/store/kv/transaction.go +++ b/store/kv/transaction.go @@ -1,5 +1,7 @@ package kv +import "golang.org/x/xerrors" + // A transaction is a list of {key, value} // dpTx implements kv.ReadableTx and kv.WritableTx @@ -13,7 +15,7 @@ type dpTx struct { func (tx *dpTx) GetBucket(name []byte) Bucket { bucket, found := tx.db[string(name)] if found { - return &dpBucket{bucket} + return bucket } return nil @@ -22,12 +24,23 @@ func (tx *dpTx) GetBucket(name []byte) Bucket { // GetBucketOrCreate implements kv.WritableTx. It creates the bucket if it does // not exist and then return it. func (tx *dpTx) GetBucketOrCreate(name []byte) (Bucket, error) { + if name == nil { + return nil, xerrors.New("create bucket failed: bucket name required") + } + + if len(name) == 0 { + return nil, xerrors.New("create bucket failed: bucket name required") + } + _, found := tx.db[string(name)] if !found { - tx.db[string(name)] = make(map[string][]byte) + tx.db[string(name)] = &dpBucket{ + make(kv), + kOrder{}, + } } - return &dpBucket{kv: tx.db[string(name)]}, nil + return tx.db[string(name)], nil } // OnCommit implements store.Transaction. It registers a callback that is called From 2da026a6f14407e30d14f0992b3c399c37d2b822 Mon Sep 17 00:00:00 2001 From: Jean Date: Wed, 8 Nov 2023 19:06:35 +0100 Subject: [PATCH 6/7] atomic and reversible transactions --- store/kv/db.go | 37 ++++++++---- store/kv/db_test.go | 124 +++++++++++++++++++++++++++++++++++---- store/kv/example_test.go | 4 +- store/kv/transaction.go | 43 ++++++++++---- 4 files changed, 171 insertions(+), 37 deletions(-) diff --git a/store/kv/db.go b/store/kv/db.go index 4549f70..99fdef5 100644 --- a/store/kv/db.go +++ b/store/kv/db.go @@ -10,16 +10,25 @@ import ( "golang.org/x/xerrors" ) -// bucket // key // value -type db map[string]*dpBucket +type privateRWMutex struct { + sync.RWMutex +} + +type bucketDb struct { + privateRWMutex + Db map[string]*dpBucket +} + +func newBucketDb() bucketDb { + return bucketDb{Db: make(map[string]*dpBucket)} +} // DB is the DELA/PURB implementation of the KV database. // // - implements kv.DB type purbDB struct { - sync.Mutex dbFilePath string - db db + db bucketDb purbIsOn bool blob *Blob } @@ -42,13 +51,11 @@ func NewDB(path string, purbIsOn bool) (DB, error) { p := &purbDB{ dbFilePath: path, - db: make(db), + db: newBucketDb(), purbIsOn: purbIsOn, blob: NewBlob(), } - p.Lock() // unlocked in Close() - buffer := bytes.NewBuffer(data) if purbIsOn { decrypted, err := p.blob.Decode(data) @@ -69,7 +76,7 @@ func NewDB(path string, purbIsOn bool) (DB, error) { // View implements kv.DB. It executes the read-only transaction in the context // of the database. func (p *purbDB) View(fn func(ReadableTx) error) error { - tx := &dpTx{db: p.db} + tx := &dpTx{db: p.db, new: newBucketDb()} err := fn(tx) @@ -83,13 +90,20 @@ func (p *purbDB) View(fn func(ReadableTx) error) error { // Update implements kv.DB. It executes the writable transaction in the context // of the database. func (p *purbDB) Update(fn func(WritableTx) error) error { - tx := &dpTx{db: p.db} + tx := &dpTx{db: p.db, new: newBucketDb()} err := fn(tx) if err != nil { return err } + p.db.Lock() + for k, v := range tx.new.Db { + p.db.Db[k] = v + } + + p.db.Unlock() + err = p.savePurbified() if err != nil { return err @@ -105,7 +119,6 @@ func (p *purbDB) Update(fn func(WritableTx) error) error { // Close implements kv.DB. It closes the database. Any view or update call will // result in an error after this function is called. func (p *purbDB) Close() error { - p.Unlock() // locked in NewDB() return nil } @@ -116,6 +129,8 @@ func (p *purbDB) serialize() (bytes.Buffer, error) { var data bytes.Buffer encoder := gob.NewEncoder(&data) + p.db.RLock() + defer p.db.RUnlock() err := encoder.Encode(p.db) return data, err } @@ -125,7 +140,7 @@ func (p *purbDB) deserialize(input *bytes.Buffer) error { err := decoder.Decode(&p.db) - for _, x := range p.db { + for _, x := range p.db.Db { x.updateIndex() } diff --git a/store/kv/db_test.go b/store/kv/db_test.go index 15b0013..4b1de3c 100644 --- a/store/kv/db_test.go +++ b/store/kv/db_test.go @@ -10,15 +10,15 @@ import ( "golang.org/x/xerrors" ) -const delaTestDir = "dela-core-Kv" +const purbDbTestDir = "purb-Db-kv" func TestPurbDB_OpenClose(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), delaTestDir) + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.db"), false) + db, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) err = db.Close() @@ -26,12 +26,12 @@ func TestPurbDB_OpenClose(t *testing.T) { } func TestPurbDB_UpdateAndView(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), delaTestDir) + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.db"), false) + db, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) ch := make(chan struct{}) @@ -64,12 +64,12 @@ func TestPurbDB_UpdateAndView(t *testing.T) { } func TestPurbTx_GetBucket(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), delaTestDir) + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.db"), false) + db, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) err = db.Update(func(tx WritableTx) error { @@ -88,12 +88,12 @@ func TestPurbTx_GetBucket(t *testing.T) { } func TestPurbBucket_Get_Set_Delete(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), delaTestDir) + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.db"), false) + db, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) err = db.Update(func(txn WritableTx) error { @@ -120,12 +120,12 @@ func TestPurbBucket_Get_Set_Delete(t *testing.T) { } func TestPurbBucket_ForEach(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), delaTestDir) + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.db"), false) + db, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) err = db.Update(func(txn WritableTx) error { @@ -147,13 +147,111 @@ func TestPurbBucket_ForEach(t *testing.T) { require.NoError(t, err) } +func TestPurbBucket_AbortedForEach(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, err := NewDB(filepath.Join(dir, "test.Db"), false) + require.NoError(t, err) + + // set some values in the DB + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + require.NoError(t, b.Set([]byte{2}, []byte{2})) + require.NoError(t, b.Set([]byte{1}, []byte{1})) + require.NoError(t, b.Set([]byte{0}, []byte{0})) + + return nil + }) + require.NoError(t, err) + + // try to alter the DB with an interrupted transaction + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + err = b.Set([]byte{0}, []byte{7}) + require.NoError(t, err) + + return xerrors.New("testing error") + }) + require.Error(t, err) + + // checks that the DB values are still ok + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + var i byte = 0 + return b.ForEach(func(k, v []byte) error { + require.Equal(t, []byte{i}, k) + require.Equal(t, []byte{i}, v) + i++ + return nil + }) + }) + require.NoError(t, err) +} + +func TestPurbBucket_ReOpenClosedDb(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, err := NewDB(filepath.Join(dir, "test.Db"), false) + require.NoError(t, err) + + // set some values in the DB + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + require.NoError(t, b.Set([]byte{2}, []byte{2})) + require.NoError(t, b.Set([]byte{1}, []byte{1})) + require.NoError(t, b.Set([]byte{0}, []byte{0})) + + return nil + }) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) + + // re-open DB file + newdb, err := NewDB(filepath.Join(dir, "test.Db"), false) + require.NoError(t, err) + + // checks that the DB values are still ok + err = newdb.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + var i byte = 0 + return b.ForEach(func(k, v []byte) error { + require.Equal(t, []byte{i}, k) + require.Equal(t, []byte{i}, v) + i++ + return nil + }) + }) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) +} + func TestPurbBucket_Scan(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), delaTestDir) + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.db"), false) + db, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) err = db.Update(func(txn WritableTx) error { diff --git a/store/kv/example_test.go b/store/kv/example_test.go index 4640a04..8935fad 100644 --- a/store/kv/example_test.go +++ b/store/kv/example_test.go @@ -14,9 +14,9 @@ func ExampleBucket_Scan() { defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "example.db"), false) + db, err := NewDB(filepath.Join(dir, "example.Db"), false) if err != nil { - panic("failed to open db: " + err.Error()) + panic("failed to open Db: " + err.Error()) } pairs := [][]byte{ diff --git a/store/kv/transaction.go b/store/kv/transaction.go index be943fc..dcaec9f 100644 --- a/store/kv/transaction.go +++ b/store/kv/transaction.go @@ -1,21 +1,39 @@ package kv -import "golang.org/x/xerrors" +import ( + "golang.org/x/exp/maps" + "golang.org/x/xerrors" +) // A transaction is a list of {key, value} // dpTx implements kv.ReadableTx and kv.WritableTx type dpTx struct { - db db + db bucketDb + new bucketDb onCommit func() } // GetBucket implements kv.ReadableTx. It returns the bucket with the given name // or nil if it does not exist. func (tx *dpTx) GetBucket(name []byte) Bucket { - bucket, found := tx.db[string(name)] + _, found := tx.new.Db[string(name)] if found { - return bucket + return tx.new.Db[string(name)] + } + + tx.db.RLock() + defer tx.db.RUnlock() + oldBucket, found := tx.db.Db[string(name)] + if found { + tx.new.Db[string(name)] = &dpBucket{ + make(kv), + kOrder{}, + } + maps.Copy(tx.new.Db[string(name)].Kv, oldBucket.Kv) + tx.new.Db[string(name)].updateIndex() + + return tx.new.Db[string(name)] } return nil @@ -32,15 +50,18 @@ func (tx *dpTx) GetBucketOrCreate(name []byte) (Bucket, error) { return nil, xerrors.New("create bucket failed: bucket name required") } - _, found := tx.db[string(name)] - if !found { - tx.db[string(name)] = &dpBucket{ - make(kv), - kOrder{}, - } + bucket := tx.GetBucket(name) + + if bucket != nil { + return bucket, nil + } + + tx.new.Db[string(name)] = &dpBucket{ + make(kv), + kOrder{}, } - return tx.db[string(name)], nil + return tx.new.Db[string(name)], nil } // OnCommit implements store.Transaction. It registers a callback that is called From 34cc2ebabb34efaf1f8b5d76a1c6f006e75d0fec Mon Sep 17 00:00:00 2001 From: Jean Date: Thu, 16 Nov 2023 17:31:28 +0100 Subject: [PATCH 7/7] add purb'd unit tests --- go.mod | 6 +- go.sum | 12 +- store/kv/blob.go | 55 +++--- store/kv/db.go | 126 +++++++++----- store/kv/db_test.go | 112 ++++++++++--- store/kv/example_test.go | 2 +- store/kv/purbdb_test.go | 349 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 560 insertions(+), 102 deletions(-) create mode 100644 store/kv/purbdb_test.go diff --git a/go.mod b/go.mod index 857f474..d6ce5f0 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,8 @@ require ( github.com/stretchr/testify v1.8.4 go.dedis.ch/dela v0.0.0-20231011144949-4677467c030c go.dedis.ch/kyber/v3 v3.1.1-0.20231024084410-31ea167adbbb - go.dedis.ch/libpurb v0.0.0-20231107160354-ffba29c17caf - golang.org/x/exp v0.0.0-20231006140011-7918f672742d + go.dedis.ch/libpurb v0.0.0-20231108133532-c70e1b84b632 + golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 ) @@ -32,7 +32,7 @@ require ( go.dedis.ch/fixbuf v1.0.3 // indirect go.etcd.io/bbolt v1.3.5 // indirect golang.org/x/crypto v0.14.0 // indirect - golang.org/x/sys v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 61f85ad..fc97be0 100644 --- a/go.sum +++ b/go.sum @@ -108,16 +108,16 @@ go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= go.dedis.ch/kyber/v3 v3.1.1-0.20231024084410-31ea167adbbb h1:G7GmitWgVMlkBaSzHJJo05bydYhLFCSPzKwCHTwf3NM= go.dedis.ch/kyber/v3 v3.1.1-0.20231024084410-31ea167adbbb/go.mod h1:nL0a5f3E//lnOCKumtzMmb8qzykxOVsihqp8C1Eb5rc= -go.dedis.ch/libpurb v0.0.0-20231107160354-ffba29c17caf h1:kNjCdcUbtc4+ANLZWApokLVzkHuW1NPxb4fNpGzFjBY= -go.dedis.ch/libpurb v0.0.0-20231107160354-ffba29c17caf/go.mod h1:XCi40g75txGSLusqJOanJytpkxiQLJzRb9RRV9UN2p4= +go.dedis.ch/libpurb v0.0.0-20231108133532-c70e1b84b632 h1:e3rOQfoyJm6ywIbuBcRwoKdWSUsPmcW9qdHO47dOmyc= +go.dedis.ch/libpurb v0.0.0-20231108133532-c70e1b84b632/go.mod h1:XCi40g75txGSLusqJOanJytpkxiQLJzRb9RRV9UN2p4= go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa h1:FRnLl4eNAQl8hwxVVC17teOw8kdjVDVAiFMtgUdTSRQ= +golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -134,8 +134,8 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= diff --git a/store/kv/blob.go b/store/kv/blob.go index 78ee15b..af08f7f 100644 --- a/store/kv/blob.go +++ b/store/kv/blob.go @@ -1,3 +1,4 @@ +// Blob is a wrapper around libpurb.Purb package kv import ( @@ -9,39 +10,31 @@ import ( "go.dedis.ch/kyber/v3/util/random" ) -type Blob struct { - purb *libpurb.Purb -} +const numberOfRecipients = 1 // NewBlob creates a new blob -func NewBlob() *Blob { - suitesInfo := getSuiteInfo() - simplified := true - +func NewBlob(keypair []key.Pair) *libpurb.Purb { p := libpurb.NewPurb( - suitesInfo, - simplified, + getSuiteInfo(), + false, random.New(), ) + p.Recipients = createRecipients(keypair) - p.Recipients = createRecipients(1) - - return &Blob{ - purb: p, - } + return p } // Encode encodes a slice of bytes into a blob -func (b *Blob) Encode(data []byte) ([]byte, error) { - err := b.purb.Encode(data) - blob := b.purb.ToBytes() +func Encode(purb *libpurb.Purb, data []byte) ([]byte, error) { + err := purb.Encode(data) + blob := purb.ToBytes() return blob, err } // Decode decodes a blob into a slice of bytes -func (b *Blob) Decode(blob []byte) ([]byte, error) { - success, decrypted, err := b.purb.Decode(blob) +func Decode(purb *libpurb.Purb, blob []byte) ([]byte, error) { + success, decrypted, err := purb.Decode(blob) if !success { err = xerrors.Errorf("Failed to decrypt blob: %v", err) @@ -71,19 +64,27 @@ func getSuiteInfo() libpurb.SuiteInfoMap { } // see example in libpurb -func createRecipients(n int) []libpurb.Recipient { - decs := make([]libpurb.Recipient, 0) +func createRecipients(keypair []key.Pair) []libpurb.Recipient { + r := make([]libpurb.Recipient, 0) suites := []libpurb.Suite{curve25519.NewBlakeSHA256Curve25519(true)} + + if len(keypair) < numberOfRecipients { + keypair = make([]key.Pair, 0) + } + for _, suite := range suites { - for i := 0; i < n; i++ { - pair := key.NewKeyPair(suite) - decs = append(decs, libpurb.Recipient{ + for i := 0; i < numberOfRecipients; i++ { + if len(keypair) < numberOfRecipients { + keypair = append(keypair, *key.NewKeyPair(suite)) + } + + r = append(r, libpurb.Recipient{ SuiteName: suite.String(), Suite: suite, - PublicKey: pair.Public, - PrivateKey: pair.Private, + PublicKey: keypair[i].Public, + PrivateKey: keypair[i].Private, }) } } - return decs + return r } diff --git a/store/kv/db.go b/store/kv/db.go index 99fdef5..dfac17f 100644 --- a/store/kv/db.go +++ b/store/kv/db.go @@ -3,10 +3,13 @@ package kv import ( "bytes" "encoding/gob" - "fmt" + "errors" + "io" "os" "sync" + "go.dedis.ch/kyber/v3/util/key" + "go.dedis.ch/libpurb/libpurb" "golang.org/x/xerrors" ) @@ -27,47 +30,70 @@ func newBucketDb() bucketDb { // // - implements kv.DB type purbDB struct { - dbFilePath string - db bucketDb - purbIsOn bool - blob *Blob + dbFile string + bucketDb bucketDb + blob *libpurb.Purb + purbIsOn bool } // NewDB opens a new database to the given file. -func NewDB(path string, purbIsOn bool) (DB, error) { +func NewDB(path string, purbIsOn bool) (DB, []key.Pair, error) { f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0755) if err != nil { - return nil, xerrors.Errorf("failed to open DB file: %v", err) + return nil, nil, xerrors.Errorf("failed to open DB file: %v", err) } defer f.Close() stats, _ := f.Stat() s := stats.Size() + if s > 0 { + return nil, nil, xerrors.New("failed to create DB file: file already exists") + } + var data = make([]byte, s) l, err := f.Read(data) - if int64(l) < s || err != nil { - return nil, xerrors.Errorf("failed to read DB file: %v", err) + if int64(l) != 0 || err != nil { + return nil, nil, xerrors.Errorf("failed to read DB file: %v", err) + } + + var b *libpurb.Purb = nil + keypair := make([]key.Pair, 0) + if purbIsOn { + b = NewBlob(nil) + pair := key.Pair{ + Public: b.Recipients[0].PublicKey, + Private: b.Recipients[0].PrivateKey, + } + keypair = append(keypair, pair) } p := &purbDB{ - dbFilePath: path, - db: newBucketDb(), - purbIsOn: purbIsOn, - blob: NewBlob(), + dbFile: path, + bucketDb: newBucketDb(), + purbIsOn: purbIsOn, + blob: b, } - buffer := bytes.NewBuffer(data) + return p, keypair, nil +} + +// LoadDB opens a database from a given file. +func LoadDB(path string, purbIsOn bool, keypair []key.Pair) (DB, error) { + var b *libpurb.Purb = nil if purbIsOn { - decrypted, err := p.blob.Decode(data) - if err != nil { - return nil, err - } - buffer.Write(decrypted) + b = NewBlob(keypair) } - err = p.deserialize(buffer) - if fmt.Sprint(err) != "EOF" { - return nil, xerrors.Errorf("failed to initialize new DB file: %v", err) + p := &purbDB{ + dbFile: path, + bucketDb: newBucketDb(), + purbIsOn: purbIsOn, + blob: b, + } + + err := p.load() + if err != nil { + return nil, err } return p, nil @@ -76,7 +102,7 @@ func NewDB(path string, purbIsOn bool) (DB, error) { // View implements kv.DB. It executes the read-only transaction in the context // of the database. func (p *purbDB) View(fn func(ReadableTx) error) error { - tx := &dpTx{db: p.db, new: newBucketDb()} + tx := &dpTx{db: p.bucketDb, new: newBucketDb()} err := fn(tx) @@ -90,21 +116,20 @@ func (p *purbDB) View(fn func(ReadableTx) error) error { // Update implements kv.DB. It executes the writable transaction in the context // of the database. func (p *purbDB) Update(fn func(WritableTx) error) error { - tx := &dpTx{db: p.db, new: newBucketDb()} + tx := &dpTx{db: p.bucketDb, new: newBucketDb()} err := fn(tx) if err != nil { return err } - p.db.Lock() + p.bucketDb.Lock() for k, v := range tx.new.Db { - p.db.Db[k] = v + p.bucketDb.Db[k] = v } + p.bucketDb.Unlock() - p.db.Unlock() - - err = p.savePurbified() + err = p.save() if err != nil { return err } @@ -125,46 +150,67 @@ func (p *purbDB) Close() error { // --------------------------------------------------------------------------- // helper functions -func (p *purbDB) serialize() (bytes.Buffer, error) { +func (p *purbDB) serialize() (*bytes.Buffer, error) { var data bytes.Buffer encoder := gob.NewEncoder(&data) - p.db.RLock() - defer p.db.RUnlock() - err := encoder.Encode(p.db) - return data, err + p.bucketDb.RLock() + defer p.bucketDb.RUnlock() + err := encoder.Encode(p.bucketDb.Db) + return &data, err } func (p *purbDB) deserialize(input *bytes.Buffer) error { decoder := gob.NewDecoder(input) - err := decoder.Decode(&p.db) + err := decoder.Decode(&p.bucketDb.Db) - for _, x := range p.db.Db { + for _, x := range p.bucketDb.Db { x.updateIndex() } return err } -func (p *purbDB) savePurbified() error { +func (p *purbDB) save() error { data, err := p.serialize() if err != nil { return xerrors.Errorf("failed to serialize DB file: %v", err) } if p.purbIsOn { - blob, err := p.blob.Encode(data.Bytes()) + blob, err := Encode(p.blob, data.Bytes()) if err != nil { return xerrors.Errorf("failed to purbify DB file: %v", err) } - data.Write(blob) + data = bytes.NewBuffer(blob) } - err = os.WriteFile(p.dbFilePath, data.Bytes(), 0755) + err = os.WriteFile(p.dbFile, data.Bytes(), 0755) if err != nil { return xerrors.Errorf("failed to save DB file: %v", err) } return nil } + +func (p *purbDB) load() error { + data, err := os.ReadFile(p.dbFile) + if err != nil { + return xerrors.Errorf("failed to load DB from file: %v", err) + } + + if p.purbIsOn && len(data) > 0 { + data, err = Decode(p.blob, data) + if err != nil { + return xerrors.Errorf("failed to decode purbified DB file: %v", err) + } + } + + buffer := bytes.NewBuffer(data) + err = p.deserialize(buffer) + if err != nil && errors.Is(err, io.EOF) { + return nil + } + return err +} diff --git a/store/kv/db_test.go b/store/kv/db_test.go index 4b1de3c..715a0eb 100644 --- a/store/kv/db_test.go +++ b/store/kv/db_test.go @@ -10,28 +10,46 @@ import ( "golang.org/x/xerrors" ) -const purbDbTestDir = "purb-Db-kv" +const dbTestDir = "db-kv" -func TestPurbDB_OpenClose(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) +func TestDb_OpenClose(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), dbTestDir) + require.NoError(t, err) + defer os.RemoveAll(dir) + + db, _, err := NewDB(filepath.Join(dir, "test.Db"), false) + require.NoError(t, err) + + err = db.Close() require.NoError(t, err) +} +func TestDb_OpenCloseReopen(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), dbTestDir) + require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.Db"), false) + db, _, err := NewDB(filepath.Join(dir, "test.Db"), false) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) + + //reopen + db, err = LoadDB(filepath.Join(dir, "test.Db"), false, nil) require.NoError(t, err) err = db.Close() require.NoError(t, err) } -func TestPurbDB_UpdateAndView(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) +func TestDb_UpdateAndView(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), dbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.Db"), false) + db, _, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) ch := make(chan struct{}) @@ -63,13 +81,13 @@ func TestPurbDB_UpdateAndView(t *testing.T) { require.NoError(t, err) } -func TestPurbTx_GetBucket(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) +func TestDb_GetBucket(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), dbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.Db"), false) + db, _, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) err = db.Update(func(tx WritableTx) error { @@ -87,13 +105,13 @@ func TestPurbTx_GetBucket(t *testing.T) { require.NoError(t, err) } -func TestPurbBucket_Get_Set_Delete(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) +func TestDb_GetSetDelete(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), dbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.Db"), false) + db, _, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) err = db.Update(func(txn WritableTx) error { @@ -119,13 +137,57 @@ func TestPurbBucket_Get_Set_Delete(t *testing.T) { require.NoError(t, err) } -func TestPurbBucket_ForEach(t *testing.T) { +func TestDb_SetReopenGet(t *testing.T) { dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.Db"), false) + db, keypair, err := NewDB(filepath.Join(dir, "test.Db"), false) + require.NoError(t, err) + + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("bucket")) + require.NoError(t, err) + + err = b.Set([]byte("ping"), []byte("pong")) + require.NoError(t, err) + + return nil + }) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) + + //reopen + db, err = LoadDB(filepath.Join(dir, "test.Db"), false, keypair) + require.NoError(t, err) + + err = db.Update(func(txn WritableTx) error { + b := txn.GetBucket([]byte("bucket")) + require.NotNil(t, b) + + value := b.Get([]byte("ping")) + require.Equal(t, []byte("pong"), value) + + return nil + }) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) + + require.NoError(t, err) +} + +func TestDb_ForEach(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), dbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, _, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) err = db.Update(func(txn WritableTx) error { @@ -147,13 +209,13 @@ func TestPurbBucket_ForEach(t *testing.T) { require.NoError(t, err) } -func TestPurbBucket_AbortedForEach(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) +func TestDb_ForEachAborted(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), dbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.Db"), false) + db, _, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) // set some values in the DB @@ -197,13 +259,13 @@ func TestPurbBucket_AbortedForEach(t *testing.T) { require.NoError(t, err) } -func TestPurbBucket_ReOpenClosedDb(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) +func TestDb_ReOpenClosedDb(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), dbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.Db"), false) + db, _, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) // set some values in the DB @@ -223,7 +285,7 @@ func TestPurbBucket_ReOpenClosedDb(t *testing.T) { require.NoError(t, err) // re-open DB file - newdb, err := NewDB(filepath.Join(dir, "test.Db"), false) + newdb, err := LoadDB(filepath.Join(dir, "test.Db"), false, nil) require.NoError(t, err) // checks that the DB values are still ok @@ -245,13 +307,13 @@ func TestPurbBucket_ReOpenClosedDb(t *testing.T) { require.NoError(t, err) } -func TestPurbBucket_Scan(t *testing.T) { - dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) +func TestDb_Scan(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), dbTestDir) require.NoError(t, err) defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "test.Db"), false) + db, _, err := NewDB(filepath.Join(dir, "test.Db"), false) require.NoError(t, err) err = db.Update(func(txn WritableTx) error { diff --git a/store/kv/example_test.go b/store/kv/example_test.go index 8935fad..fb3354e 100644 --- a/store/kv/example_test.go +++ b/store/kv/example_test.go @@ -14,7 +14,7 @@ func ExampleBucket_Scan() { defer os.RemoveAll(dir) - db, err := NewDB(filepath.Join(dir, "example.Db"), false) + db, _, err := NewDB(filepath.Join(dir, "example.Db"), false) if err != nil { panic("failed to open Db: " + err.Error()) } diff --git a/store/kv/purbdb_test.go b/store/kv/purbdb_test.go new file mode 100644 index 0000000..37ece42 --- /dev/null +++ b/store/kv/purbdb_test.go @@ -0,0 +1,349 @@ +package kv + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" +) + +const purbDbTestDir = "purb-db-kv" + +func TestPurbDb_OpenClose(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + defer os.RemoveAll(dir) + + db, _, err := NewDB(filepath.Join(dir, "test.Db"), true) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) +} + +func TestPurbDb_OpenCloseReopen(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + defer os.RemoveAll(dir) + + db, keypair, err := NewDB(filepath.Join(dir, "test.Db"), true) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) + + //reopen + db, err = LoadDB(filepath.Join(dir, "test.Db"), true, keypair) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) +} + +func TestPurbDb_UpdateAndView(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, _, err := NewDB(filepath.Join(dir, "test.Db"), true) + require.NoError(t, err) + + ch := make(chan struct{}) + err = db.Update(func(txn WritableTx) error { + txn.OnCommit(func() { close(ch) }) + + bucket, err := txn.GetBucketOrCreate([]byte("bucket")) + require.NoError(t, err) + + return bucket.Set([]byte("ping"), []byte("pong")) + }) + require.NoError(t, err) + + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal("timeout") + } + + err = db.View(func(txn ReadableTx) error { + bucket := txn.GetBucket([]byte("bucket")) + require.NotNil(t, bucket) + + value := bucket.Get([]byte("ping")) + require.Equal(t, []byte("pong"), value) + + return nil + }) + require.NoError(t, err) +} + +func TestPurbDb_GetBucket(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, _, err := NewDB(filepath.Join(dir, "test.Db"), true) + require.NoError(t, err) + + err = db.Update(func(tx WritableTx) error { + require.Nil(t, tx.GetBucket([]byte("unknown"))) + + _, err := tx.GetBucketOrCreate([]byte("A")) + require.NoError(t, err) + require.NotNil(t, tx.GetBucket([]byte("A"))) + + _, err = tx.GetBucketOrCreate(nil) + require.EqualError(t, err, "create bucket failed: bucket name required") + + return nil + }) + require.NoError(t, err) +} + +func TestPurbDb_GetSetDelete(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, _, err := NewDB(filepath.Join(dir, "test.Db"), true) + require.NoError(t, err) + + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("bucket")) + require.NoError(t, err) + + require.NoError(t, b.Set([]byte("ping"), []byte("pong"))) + + value := b.Get([]byte("ping")) + require.Equal(t, []byte("pong"), value) + + value = b.Get([]byte("pong")) + require.Nil(t, value) + + require.NoError(t, b.Delete([]byte("ping"))) + + value = b.Get([]byte("ping")) + require.Nil(t, value) + + return nil + }) + + require.NoError(t, err) +} + +func TestPurbDb_SetReopenGet(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, keypair, err := NewDB(filepath.Join(dir, "test.Db"), true) + require.NoError(t, err) + + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("bucket")) + require.NoError(t, err) + + err = b.Set([]byte("ping"), []byte("pong")) + require.NoError(t, err) + + return nil + }) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) + + //reopen + db, err = LoadDB(filepath.Join(dir, "test.Db"), true, keypair) + require.NoError(t, err) + + err = db.Update(func(txn WritableTx) error { + b := txn.GetBucket([]byte("bucket")) + require.NotNil(t, b) + + value := b.Get([]byte("ping")) + require.Equal(t, []byte("pong"), value) + + return nil + }) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) + + require.NoError(t, err) +} + +func TestPurbDb_ForEach(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, _, err := NewDB(filepath.Join(dir, "test.Db"), true) + require.NoError(t, err) + + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + require.NoError(t, b.Set([]byte{2}, []byte{2})) + require.NoError(t, b.Set([]byte{1}, []byte{1})) + require.NoError(t, b.Set([]byte{0}, []byte{0})) + + var i byte = 0 + return b.ForEach(func(k, v []byte) error { + require.Equal(t, []byte{i}, k) + require.Equal(t, []byte{i}, v) + i++ + return nil + }) + }) + require.NoError(t, err) +} + +func TestPurbDb_ForEachAborted(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, _, err := NewDB(filepath.Join(dir, "test.Db"), true) + require.NoError(t, err) + + // set some values in the DB + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + require.NoError(t, b.Set([]byte{2}, []byte{2})) + require.NoError(t, b.Set([]byte{1}, []byte{1})) + require.NoError(t, b.Set([]byte{0}, []byte{0})) + + return nil + }) + require.NoError(t, err) + + // try to alter the DB with an interrupted transaction + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + err = b.Set([]byte{0}, []byte{7}) + require.NoError(t, err) + + return xerrors.New("testing error") + }) + require.Error(t, err) + + // checks that the DB values are still ok + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + var i byte = 0 + return b.ForEach(func(k, v []byte) error { + require.Equal(t, []byte{i}, k) + require.Equal(t, []byte{i}, v) + i++ + return nil + }) + }) + require.NoError(t, err) +} + +func TestPurbDb_ReOpenClosedDb(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, keypair, err := NewDB(filepath.Join(dir, "test.Db"), true) + require.NoError(t, err) + + // set some values in the DB + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + require.NoError(t, b.Set([]byte{2}, []byte{2})) + require.NoError(t, b.Set([]byte{1}, []byte{1})) + require.NoError(t, b.Set([]byte{0}, []byte{0})) + + return nil + }) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) + + // re-open DB file + newdb, err := LoadDB(filepath.Join(dir, "test.Db"), true, keypair) + require.NoError(t, err) + + // checks that the DB values are still ok + err = newdb.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("test")) + require.NoError(t, err) + + var i byte = 0 + return b.ForEach(func(k, v []byte) error { + require.Equal(t, []byte{i}, k) + require.Equal(t, []byte{i}, v) + i++ + return nil + }) + }) + require.NoError(t, err) + + err = db.Close() + require.NoError(t, err) +} + +func TestPurbDb_Scan(t *testing.T) { + dir, err := os.MkdirTemp(os.TempDir(), purbDbTestDir) + require.NoError(t, err) + + defer os.RemoveAll(dir) + + db, _, err := NewDB(filepath.Join(dir, "test.Db"), true) + require.NoError(t, err) + + err = db.Update(func(txn WritableTx) error { + b, err := txn.GetBucketOrCreate([]byte("bucket")) + require.NoError(t, err) + + require.NoError(t, b.Set([]byte{7}, []byte{7})) + require.NoError(t, b.Set([]byte{0}, []byte{0})) + + var i byte = 0 + err = b.Scan(nil, func(k, v []byte) error { + require.Equal(t, []byte{i}, k) + require.Equal(t, []byte{i}, v) + i += 7 + return nil + }) + require.NoError(t, err) + require.Equal(t, byte(14), i) + + err = b.Scan([]byte{1}, func(k, v []byte) error { + return xerrors.New("callback error") + }) + require.NoError(t, err) + + err = b.Scan([]byte{}, func(k, v []byte) error { + return xerrors.New("callback error") + }) + require.ErrorContains(t, err, "callback error") + + return nil + }) + require.NoError(t, err) +}