Skip to content
This repository has been archived by the owner on Jun 20, 2024. It is now read-only.

refactor: use IPIP-402 param names when GRAPH_BACKEND=true #108

Merged
merged 1 commit into from
May 17, 2023
Merged

refactor: use IPIP-402 param names when GRAPH_BACKEND=true #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions docs/environment-variables.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,16 @@ The size of in-memory [2Q cache](https://pkg.go.dev/github.com/hashicorp/golang-

Default: `false`

When set to `true`, requests to backend will use
`?format=car&car-scope=..&bytes=..` in addition to `?format=raw` to reduce the
number of round trips.

This is an experimental feature that depends on `&car-scope=..&bytes=..`
parameters. Currently only `https://l1s.strn.pl` supports it, but our
intention is to standardize it and add it to the
[trustless gateway spec](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
When set to `true`, requests to backend will use `?format=car` in addition to
`?format=raw` to reduce the number of round trips.

This is an experimental feature that depends on pathing, `dag-scope` and `entity-bytes`
parameters from [IPIP-402](https://github.com/ipfs/specs/pull/402).

Currently only `https://l1s.strn.pl` supports it, but our intention is to
standardize it ([IPIP-402](https://github.com/ipfs/specs/pull/402)) and add it
to the [trustless gateway spec](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
and [boxo/gateway](https://github.com/ipfs/boxo/pull/303) reference library
in the near feature.

## Proxy Backend
Expand Down
41 changes: 26 additions & 15 deletions lib/graph_gateway.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics {
Subsystem: "gw_graph_backend",
Name: "car_fetch_params",
Help: "How many times specific CAR parameter was used during CAR data fetch.",
}, []string{"carScope", "ranges"}) // we use 'ranges' instead of 'bytes' here because we only caount the number of ranges present
}, []string{"dagScope", "entityRanges"}) // we use 'ranges' instead of 'bytes' here because we only count the number of ranges present
prometheus.MustRegister(carParamsMetric)

bytesRangeStartMetric := prometheus.NewHistogram(prometheus.HistogramOpts{
Expand Down Expand Up @@ -426,18 +426,18 @@ func wrapNodeWithClose[T files.Node](node T, closeFn func()) (T, error) {

func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, byteRanges ...gateway.ByteRange) (gateway.ContentPathMetadata, *gateway.GetResponse, error) {
rangeCount := len(byteRanges)
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "file", "ranges": strconv.Itoa(rangeCount)}).Inc()
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": strconv.Itoa(rangeCount)}).Inc()

// TODO: remove &depth= from all CAR request after transition is done:
// TODO: remove &bytes= &depth= and &car-scope from all CAR request after transition is done:
// https://github.com/ipfs/bifrost-gateway/issues/80
carParams := "?format=car&car-scope=file&depth=1"
carParams := "?format=car&dag-scope=entity&car-scope=file&depth=1"

// fetch CAR with &bytes= to get minimal set of blocks for the request
// Note: majority of requests have 0 or max 1 ranges. if there are more ranges than one,
// that is a niche edge cache we don't prefetch as CAR and use fallback blockstore instead.
if rangeCount > 0 {
bytesBuilder := strings.Builder{}
bytesBuilder.WriteString("&bytes=")
bytesBuilder.WriteString("&entity-bytes=")
r := byteRanges[0]

bytesBuilder.WriteString(strconv.FormatUint(r.From, 10))
Expand All @@ -451,7 +451,18 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by

// TODO: move to boxo or to loadRequestIntoSharedBlockstoreAndBlocksGateway after we pass params in a humane way
api.metrics.bytesRangeSizeMetric.Observe(float64(*r.To) - float64(r.From) + 1)
} else {
bytesBuilder.WriteString("*")
}

// TODO: &bytes= is present only for transition reasons, remove below block after https://github.com/ipfs/bifrost-gateway/issues/80 is done
if strings.HasSuffix(carParams, "&depth=1") {
parts := strings.Split(bytesBuilder.String(), "=")
value := parts[1]
bytesBuilder.WriteString("&bytes=")
bytesBuilder.WriteString(value)
}

carParams += bytesBuilder.String()

}
Expand All @@ -471,8 +482,8 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by
}

func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "all", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=all&depth=all")
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all&car-scope=all&depth=all")
if err != nil {
return gateway.ContentPathMetadata{}, nil, err
}
Expand All @@ -488,9 +499,9 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath)
}

func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "block", "ranges": "0"}).Inc()
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc()
// TODO: if path is `/ipfs/cid`, we should use ?format=raw
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=block&depth=0")
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block&car-scope=block&depth=0")
if err != nil {
return gateway.ContentPathMetadata{}, nil, err
}
Expand All @@ -506,13 +517,13 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat
}

func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "file", "ranges": "1"}).Inc()
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": "1"}).Inc()

// TODO: we probably want to move this either to boxo, or at least to loadRequestIntoSharedBlockstoreAndBlocksGateway
api.metrics.bytesRangeStartMetric.Observe(0)
api.metrics.bytesRangeSizeMetric.Observe(1024)

blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=file&depth=1&bytes=0:1023")
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=entity&entity-bytes=0:1023&car-scope=file&depth=1&bytes=0:1023")

if err != nil {
return gateway.ContentPathMetadata{}, nil, err
Expand All @@ -529,8 +540,8 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (
}

func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "block", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=block&depth=0")
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block&car-scope=block&depth=0")
if err != nil {
return gateway.ContentPathMetadata{}, err
}
Expand All @@ -539,8 +550,8 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable
}

func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "all", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=all&depth=all")
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all&car-scope=all&depth=all")
if err != nil {
return gateway.ContentPathMetadata{}, nil, nil, err
}
Expand Down