Skip to content

Commit

Permalink
Merge branch 'main' into schemaFile
Browse files Browse the repository at this point in the history
  • Loading branch information
kartikaysaxena authored Jan 13, 2025
2 parents 813e627 + fabeff1 commit 26af2a9
Show file tree
Hide file tree
Showing 116 changed files with 4,850 additions and 11,779 deletions.
82 changes: 79 additions & 3 deletions .github/workflows/build-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -100,14 +100,14 @@ jobs:
- name: "Integration tests"
run: "go run mage.go test:integration"

datastore:
name: "Datastore Tests"
datastoreinttest:
name: "Datastore Integration Tests"
runs-on: "buildjet-8vcpu-ubuntu-2204"
needs: "paths-filter"
strategy:
fail-fast: false
matrix:
datastore: ["crdb", "mysql", "postgres", "spanner", "pgbouncer"]
datastore: ["crdb", "mysql", "spanner"]
steps:
- uses: "actions/checkout@v4"
if: |
Expand All @@ -125,11 +125,87 @@ jobs:
if: |
needs.paths-filter.outputs.codechange == 'true'
run: "go run mage.go testds:${{ matrix.datastore }}"

datastoreconstest:
name: "Datastore Consistency Tests"
runs-on: "buildjet-4vcpu-ubuntu-2204"
needs: "paths-filter"
strategy:
fail-fast: false
matrix:
datastore: ["crdb", "mysql", "spanner"]
steps:
- uses: "actions/checkout@v4"
if: |
needs.paths-filter.outputs.codechange == 'true'
- uses: "authzed/actions/setup-go@main"
if: |
needs.paths-filter.outputs.codechange == 'true'
- uses: "docker/login-action@v3"
if: |
needs.paths-filter.outputs.codechange == 'true'
with:
username: "${{ env.DOCKERHUB_PUBLIC_USER }}"
password: "${{ env.DOCKERHUB_PUBLIC_ACCESS_TOKEN }}"
- name: "Consistency tests"
if: |
needs.paths-filter.outputs.codechange == 'true'
run: "go run mage.go testcons:${{ matrix.datastore }}"

pgdatastoreinttest:
name: "Datastore Integration Tests"
runs-on: "buildjet-4vcpu-ubuntu-2204"
needs: "paths-filter"
strategy:
fail-fast: false
matrix:
datastore: ["postgres", "pgbouncer"]
pgversion: ["13.8", "14", "15", "16", "17"]
steps:
- uses: "actions/checkout@v4"
if: |
needs.paths-filter.outputs.codechange == 'true'
- uses: "authzed/actions/setup-go@main"
if: |
needs.paths-filter.outputs.codechange == 'true'
- uses: "docker/login-action@v3"
if: |
needs.paths-filter.outputs.codechange == 'true'
with:
username: "${{ env.DOCKERHUB_PUBLIC_USER }}"
password: "${{ env.DOCKERHUB_PUBLIC_ACCESS_TOKEN }}"
- name: "Integration tests"
if: |
needs.paths-filter.outputs.codechange == 'true'
run: "go run mage.go testds:${{ matrix.datastore }}ver ${{ matrix.pgversion }}"

pgdatastoreconstest:
name: "Datastore Consistency Tests"
runs-on: "buildjet-4vcpu-ubuntu-2204"
needs: "paths-filter"
strategy:
fail-fast: false
matrix:
datastore: ["postgres"]
pgversion: ["13.8", "14", "15", "16", "17"]
steps:
- uses: "actions/checkout@v4"
if: |
needs.paths-filter.outputs.codechange == 'true'
- uses: "authzed/actions/setup-go@main"
if: |
needs.paths-filter.outputs.codechange == 'true'
- uses: "docker/login-action@v3"
if: |
needs.paths-filter.outputs.codechange == 'true'
with:
username: "${{ env.DOCKERHUB_PUBLIC_USER }}"
password: "${{ env.DOCKERHUB_PUBLIC_ACCESS_TOKEN }}"
- name: "Consistency tests"
if: |
needs.paths-filter.outputs.codechange == 'true'
run: "go run mage.go testcons:postgresver ${{ matrix.pgversion }}"

e2e:
name: "E2E"
runs-on: "buildjet-8vcpu-ubuntu-2204"
Expand Down
4 changes: 2 additions & 2 deletions cmd/spicedb/servetesting_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ type spicedbHandle struct {
cleanup func()
}

const retryCount = 5
const retryCount = 8

func newTester(t *testing.T, containerOpts *dockertest.RunOptions, token string, withExistingSchema bool) (*spicedbHandle, error) {
for i := 0; i < retryCount; i++ {
Expand All @@ -186,7 +186,7 @@ func newTester(t *testing.T, containerOpts *dockertest.RunOptions, token string,
return nil, fmt.Errorf("could not connect to docker: %w", err)
}

pool.MaxWait = 30 * time.Second
pool.MaxWait = 60 * time.Second

resource, err := pool.RunWithOptions(containerOpts)
if err != nil {
Expand Down
7 changes: 4 additions & 3 deletions internal/caveats/run_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require"

"github.com/authzed/spicedb/internal/caveats"
"github.com/authzed/spicedb/internal/datastore/dsfortesting"
"github.com/authzed/spicedb/internal/datastore/memdb"
"github.com/authzed/spicedb/internal/testfixtures"
"github.com/authzed/spicedb/pkg/datastore"
Expand Down Expand Up @@ -448,7 +449,7 @@ func TestRunCaveatExpressions(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
req := require.New(t)

rawDS, err := memdb.NewMemdbDatastore(0, 0, memdb.DisableGC)
rawDS, err := dsfortesting.NewMemDBDatastoreForTesting(0, 0, memdb.DisableGC)
req.NoError(err)

ds, _ := testfixtures.DatastoreFromSchemaAndTestRelationships(rawDS, `
Expand Down Expand Up @@ -507,7 +508,7 @@ func TestRunCaveatExpressions(t *testing.T) {
func TestRunCaveatWithMissingMap(t *testing.T) {
req := require.New(t)

rawDS, err := memdb.NewMemdbDatastore(0, 0, memdb.DisableGC)
rawDS, err := dsfortesting.NewMemDBDatastoreForTesting(0, 0, memdb.DisableGC)
req.NoError(err)

ds, _ := testfixtures.DatastoreFromSchemaAndTestRelationships(rawDS, `
Expand Down Expand Up @@ -536,7 +537,7 @@ func TestRunCaveatWithMissingMap(t *testing.T) {
func TestRunCaveatWithEmptyMap(t *testing.T) {
req := require.New(t)

rawDS, err := memdb.NewMemdbDatastore(0, 0, memdb.DisableGC)
rawDS, err := dsfortesting.NewMemDBDatastoreForTesting(0, 0, memdb.DisableGC)
req.NoError(err)

ds, _ := testfixtures.DatastoreFromSchemaAndTestRelationships(rawDS, `
Expand Down
159 changes: 159 additions & 0 deletions internal/datastore/common/relationships.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
package common

import (
"context"
"database/sql"
"fmt"
"time"

"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"google.golang.org/protobuf/types/known/timestamppb"

"github.com/authzed/spicedb/pkg/datastore"
corev1 "github.com/authzed/spicedb/pkg/proto/core/v1"
"github.com/authzed/spicedb/pkg/tuple"
)

const errUnableToQueryRels = "unable to query relationships: %w"

// Querier is an interface for querying the database.
type Querier[R Rows] interface {
QueryFunc(ctx context.Context, f func(context.Context, R) error, sql string, args ...any) error
}

// Rows is a common interface for database rows reading.
type Rows interface {
Scan(dest ...any) error
Next() bool
Err() error
}

type closeRowsWithError interface {
Rows
Close() error
}

type closeRows interface {
Rows
Close()
}

// QueryRelationships queries relationships for the given query and transaction.
func QueryRelationships[R Rows, C ~map[string]any](ctx context.Context, builder RelationshipsQueryBuilder, tx Querier[R]) (datastore.RelationshipIterator, error) {
span := trace.SpanFromContext(ctx)
sqlString, args, err := builder.SelectSQL()
if err != nil {
return nil, fmt.Errorf(errUnableToQueryRels, err)
}

var resourceObjectType string
var resourceObjectID string
var resourceRelation string
var subjectObjectType string
var subjectObjectID string
var subjectRelation string
var caveatName sql.NullString
var caveatCtx C
var expiration *time.Time

var integrityKeyID string
var integrityHash []byte
var timestamp time.Time

span.AddEvent("Selecting columns")
colsToSelect, err := ColumnsToSelect(builder, &resourceObjectType, &resourceObjectID, &resourceRelation, &subjectObjectType, &subjectObjectID, &subjectRelation, &caveatName, &caveatCtx, &expiration, &integrityKeyID, &integrityHash, &timestamp)
if err != nil {
return nil, fmt.Errorf(errUnableToQueryRels, err)
}

span.AddEvent("Returning iterator", trace.WithAttributes(attribute.Int("column-count", len(colsToSelect))))
return func(yield func(tuple.Relationship, error) bool) {
span.AddEvent("Issuing query to database")
err := tx.QueryFunc(ctx, func(ctx context.Context, rows R) error {
span.AddEvent("Query issued to database")

var r Rows = rows
if crwe, ok := r.(closeRowsWithError); ok {
defer LogOnError(ctx, crwe.Close)
} else if cr, ok := r.(closeRows); ok {
defer cr.Close()
}

relCount := 0
for rows.Next() {
if relCount == 0 {
span.AddEvent("First row returned")
}

if err := rows.Scan(colsToSelect...); err != nil {
return fmt.Errorf(errUnableToQueryRels, fmt.Errorf("scan err: %w", err))
}

if relCount == 0 {
span.AddEvent("First row scanned")
}

var caveat *corev1.ContextualizedCaveat
if !builder.SkipCaveats || builder.Schema.ColumnOptimization == ColumnOptimizationOptionNone {
if caveatName.Valid {
var err error
caveat, err = ContextualizedCaveatFrom(caveatName.String, caveatCtx)
if err != nil {
return fmt.Errorf(errUnableToQueryRels, fmt.Errorf("unable to fetch caveat context: %w", err))
}
}
}

var integrity *corev1.RelationshipIntegrity
if integrityKeyID != "" {
integrity = &corev1.RelationshipIntegrity{
KeyId: integrityKeyID,
Hash: integrityHash,
HashedAt: timestamppb.New(timestamp),
}
}

if expiration != nil {
// Ensure the expiration is always read in UTC, since some datastores (like CRDB)
// will normalize to local time.
t := expiration.UTC()
expiration = &t
}

relCount++
if !yield(tuple.Relationship{
RelationshipReference: tuple.RelationshipReference{
Resource: tuple.ObjectAndRelation{
ObjectType: resourceObjectType,
ObjectID: resourceObjectID,
Relation: resourceRelation,
},
Subject: tuple.ObjectAndRelation{
ObjectType: subjectObjectType,
ObjectID: subjectObjectID,
Relation: subjectRelation,
},
},
OptionalCaveat: caveat,
OptionalExpiration: expiration,
OptionalIntegrity: integrity,
}, nil) {
return nil
}
}

span.AddEvent("Relationships loaded", trace.WithAttributes(attribute.Int("relCount", relCount)))
if err := rows.Err(); err != nil {
return fmt.Errorf(errUnableToQueryRels, fmt.Errorf("rows err: %w", err))
}

return nil
}, sqlString, args...)
if err != nil {
if !yield(tuple.Relationship{}, err) {
return
}
}
}, nil
}
Loading

0 comments on commit 26af2a9

Please sign in to comment.