Skip to content
This repository has been archived by the owner on Jun 20, 2024. It is now read-only.

Commit

Permalink
feat: support IPIP-402 (#108)
Browse files Browse the repository at this point in the history 
follows transition plan to send all versions until every system
(lassie, l1) upgrades.
  • Loading branch information
@lidel
lidel authored May 17, 2023
1 parent 13fa459 commit f9df8a0
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 23 deletions.
18 changes: 10 additions & 8 deletions docs/environment-variables.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,16 @@ The size of in-memory [2Q cache](https://pkg.go.dev/github.com/hashicorp/golang-

Default: `false`

When set to `true`, requests to backend will use
`?format=car&car-scope=..&bytes=..` in addition to `?format=raw` to reduce the
number of round trips.

This is an experimental feature that depends on `&car-scope=..&bytes=..`
parameters. Currently only `https://l1s.strn.pl` supports it, but our
intention is to standardize it and add it to the
[trustless gateway spec](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
When set to `true`, requests to backend will use `?format=car` in addition to
`?format=raw` to reduce the number of round trips.

This is an experimental feature that depends on pathing, `dag-scope` and `entity-bytes`
parameters from [IPIP-402](https://github.com/ipfs/specs/pull/402).

Currently only `https://l1s.strn.pl` supports it, but our intention is to
standardize it ([IPIP-402](https://github.com/ipfs/specs/pull/402)) and add it
to the [trustless gateway spec](https://specs.ipfs.tech/http-gateways/trustless-gateway/)
and [boxo/gateway](https://github.com/ipfs/boxo/pull/303) reference library
in the near feature.

## Proxy Backend
Expand Down
41 changes: 26 additions & 15 deletions lib/graph_gateway.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func registerGraphGatewayMetrics() *GraphGatewayMetrics {
Subsystem: "gw_graph_backend",
Name: "car_fetch_params",
Help: "How many times specific CAR parameter was used during CAR data fetch.",
}, []string{"carScope", "ranges"}) // we use 'ranges' instead of 'bytes' here because we only caount the number of ranges present
}, []string{"dagScope", "entityRanges"}) // we use 'ranges' instead of 'bytes' here because we only count the number of ranges present
prometheus.MustRegister(carParamsMetric)

bytesRangeStartMetric := prometheus.NewHistogram(prometheus.HistogramOpts{
Expand Down Expand Up @@ -426,18 +426,18 @@ func wrapNodeWithClose[T files.Node](node T, closeFn func()) (T, error) {

func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, byteRanges ...gateway.ByteRange) (gateway.ContentPathMetadata, *gateway.GetResponse, error) {
rangeCount := len(byteRanges)
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "file", "ranges": strconv.Itoa(rangeCount)}).Inc()
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": strconv.Itoa(rangeCount)}).Inc()

// TODO: remove &depth= from all CAR request after transition is done:
// TODO: remove &bytes= &depth= and &car-scope from all CAR request after transition is done:
// https://github.com/ipfs/bifrost-gateway/issues/80
carParams := "?format=car&car-scope=file&depth=1"
carParams := "?format=car&dag-scope=entity&car-scope=file&depth=1"

// fetch CAR with &bytes= to get minimal set of blocks for the request
// Note: majority of requests have 0 or max 1 ranges. if there are more ranges than one,
// that is a niche edge cache we don't prefetch as CAR and use fallback blockstore instead.
if rangeCount > 0 {
bytesBuilder := strings.Builder{}
bytesBuilder.WriteString("&bytes=")
bytesBuilder.WriteString("&entity-bytes=")
r := byteRanges[0]

bytesBuilder.WriteString(strconv.FormatUint(r.From, 10))
Expand All @@ -451,7 +451,18 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by

// TODO: move to boxo or to loadRequestIntoSharedBlockstoreAndBlocksGateway after we pass params in a humane way
api.metrics.bytesRangeSizeMetric.Observe(float64(*r.To) - float64(r.From) + 1)
} else {
bytesBuilder.WriteString("*")
}

// TODO: &bytes= is present only for transition reasons, remove below block after https://github.com/ipfs/bifrost-gateway/issues/80 is done
if strings.HasSuffix(carParams, "&depth=1") {
parts := strings.Split(bytesBuilder.String(), "=")
value := parts[1]
bytesBuilder.WriteString("&bytes=")
bytesBuilder.WriteString(value)
}

carParams += bytesBuilder.String()

}
Expand All @@ -471,8 +482,8 @@ func (api *GraphGateway) Get(ctx context.Context, path gateway.ImmutablePath, by
}

func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "all", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=all&depth=all")
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all&car-scope=all&depth=all")
if err != nil {
return gateway.ContentPathMetadata{}, nil, err
}
Expand All @@ -488,9 +499,9 @@ func (api *GraphGateway) GetAll(ctx context.Context, path gateway.ImmutablePath)
}

func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "block", "ranges": "0"}).Inc()
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc()
// TODO: if path is `/ipfs/cid`, we should use ?format=raw
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=block&depth=0")
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block&car-scope=block&depth=0")
if err != nil {
return gateway.ContentPathMetadata{}, nil, err
}
Expand All @@ -506,13 +517,13 @@ func (api *GraphGateway) GetBlock(ctx context.Context, path gateway.ImmutablePat
}

func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "file", "ranges": "1"}).Inc()
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "entity", "entityRanges": "1"}).Inc()

// TODO: we probably want to move this either to boxo, or at least to loadRequestIntoSharedBlockstoreAndBlocksGateway
api.metrics.bytesRangeStartMetric.Observe(0)
api.metrics.bytesRangeSizeMetric.Observe(1024)

blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=file&depth=1&bytes=0:1023")
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=entity&entity-bytes=0:1023&car-scope=file&depth=1&bytes=0:1023")

if err != nil {
return gateway.ContentPathMetadata{}, nil, err
Expand All @@ -529,8 +540,8 @@ func (api *GraphGateway) Head(ctx context.Context, path gateway.ImmutablePath) (
}

func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "block", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=block&depth=0")
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "block", "entityRanges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=block&car-scope=block&depth=0")
if err != nil {
return gateway.ContentPathMetadata{}, err
}
Expand All @@ -539,8 +550,8 @@ func (api *GraphGateway) ResolvePath(ctx context.Context, path gateway.Immutable
}

func (api *GraphGateway) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
api.metrics.carParamsMetric.With(prometheus.Labels{"carScope": "all", "ranges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&car-scope=all&depth=all")
api.metrics.carParamsMetric.With(prometheus.Labels{"dagScope": "all", "entityRanges": "0"}).Inc()
blkgw, closeFn, err := api.loadRequestIntoSharedBlockstoreAndBlocksGateway(ctx, path.String()+"?format=car&dag-scope=all&car-scope=all&depth=all")
if err != nil {
return gateway.ContentPathMetadata{}, nil, nil, err
}
Expand Down

0 comments on commit f9df8a0

Please sign in to comment.