diff --git a/go.mod b/go.mod index e57c5007..93158ba0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/filecoin-project/lassie -go 1.21 +go 1.22 require ( github.com/benbjohnson/clock v1.3.5 diff --git a/pkg/internal/itest/e2e_test.go b/pkg/internal/itest/e2e_test.go index 41e70a80..b5592253 100644 --- a/pkg/internal/itest/e2e_test.go +++ b/pkg/internal/itest/e2e_test.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "time" @@ -54,11 +55,14 @@ func TestTrustlessGatewayE2E(t *testing.T) { req.NoError(err) // install the indexer to announce to + t.Log("Installing indexer") indexer := filepath.Join(tr.Dir, "storetheindex") - tr.Run("go", "install", "github.com/ipni/storetheindex@latest") + tr.Run("go", "install", "github.com/ipni/storetheindex@v0.8.14") // install the ipni cli to inspect the indexer + t.Log("Installing ipni") ipni := filepath.Join(tr.Dir, "ipni") - tr.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@latest") + tr.Run("go", "install", "github.com/ipni/ipni-cli/cmd/ipni@v0.1.8") + t.Log("Installing frisbii") // install frisbii to serve the content frisbii := filepath.Join(tr.Dir, "frisbii") tr.Run("go", "install", "github.com/ipld/frisbii/cmd/frisbii@latest") @@ -67,7 +71,10 @@ func TestTrustlessGatewayE2E(t *testing.T) { req.NoError(err) // initialise and start the indexer and adjust the config + t.Log("Initialising indexer") tr.Run(indexer, "init", "--store", "pebble", "--pubsub-topic", "/indexer/ingest/mainnet", "--no-bootstrap") + + t.Log("Starting indexer") indexerReady := test.NewStdoutWatcher(test.IndexerReadyMatch) cmdIndexer := tr.Start(test.NewExecution(indexer, "daemon").WithWatcher(indexerReady)) select { @@ -82,6 +89,7 @@ func TestTrustlessGatewayE2E(t *testing.T) { carPath := trustlesspathing.Unixfs20mVarietyCARPath() // start frisbii with the fixture CAR + t.Logf("Starting frisbii with CAR [%s] and root [%s]", carPath, root) frisbiiReady := test.NewStdoutWatcher("Announce() complete") cmdFrisbii := tr.Start(test.NewExecution(frisbii, "--listen", "localhost:37471", @@ -101,8 +109,6 @@ func TestTrustlessGatewayE2E(t *testing.T) { req.Eventually(func() bool { mh := root.Hash().B58String() findOutput := tr.Run(ipni, "find", "--no-priv", "-i", "http://localhost:3000", "-mh", mh) - t.Logf("import output:\n%s\n", findOutput) - if bytes.Contains(findOutput, []byte("not found")) { return false } @@ -110,8 +116,6 @@ func TestTrustlessGatewayE2E(t *testing.T) { t.Logf("mh %s: unexpected error: %s", mh, findOutput) return false } - - t.Logf("mh %s: found", mh) return true }, 10*time.Second, time.Second) @@ -172,6 +176,7 @@ func TestTrustlessGatewayE2E(t *testing.T) { if testCase.Path != "" { args[len(args)-1] = args[len(args)-1] + "/" + testCase.Path } + t.Logf("Running lassie %s", strings.Join(args, " ")) tr.Run(lassie, args...) _, err = os.Stat(expectedCarPath) @@ -188,6 +193,8 @@ func TestTrustlessGatewayE2E(t *testing.T) { reqUrl, err := url.Parse("http://localhost:30000/" + testCase.AsQuery()) req.NoError(err) + t.Logf("Fetching %s", reqUrl.String()) + // download and read all body from URL along with Accept:application/vnd.ipld.car header reqReq, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl.String(), nil) req.NoError(err) diff --git a/pkg/server/http/ipfs.go b/pkg/server/http/ipfs.go index d67ea47d..ffcb6547 100644 --- a/pkg/server/http/ipfs.go +++ b/pkg/server/http/ipfs.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "net/url" "strconv" "strings" @@ -24,13 +25,18 @@ import ( func IpfsHandler(fetcher types.Fetcher, cfg HttpServerConfig) func(http.ResponseWriter, *http.Request) { return func(res http.ResponseWriter, req *http.Request) { - statusLogger := newStatusLogger(req.Method, req.URL.Path) + unescapedPath, err := url.PathUnescape(req.URL.Path) + if err != nil { + logger.Warnf("error unescaping path: %s", err) + unescapedPath = req.URL.Path + } + statusLogger := newStatusLogger(req.Method, unescapedPath) if !checkGet(req, res, statusLogger) { return } - ok, request := decodeRetrievalRequest(cfg, res, req, statusLogger) + ok, request := decodeRetrievalRequest(cfg, res, req, unescapedPath, statusLogger) if !ok { return } @@ -87,7 +93,7 @@ func IpfsHandler(fetcher types.Fetcher, cfg HttpServerConfig) func(http.Response res.Header().Set("Content-Type", trustlesshttp.DefaultContentType().WithDuplicates(request.Duplicates).String()) res.Header().Set("Etag", request.Etag()) res.Header().Set("X-Content-Type-Options", "nosniff") - res.Header().Set("X-Ipfs-Path", trustlessutils.PathEscape(req.URL.Path)) + res.Header().Set("X-Ipfs-Path", trustlessutils.PathEscape(unescapedPath)) res.Header().Set("X-Trace-Id", requestId) statusLogger.logStatus(200, "OK") close(bytesWritten) @@ -152,8 +158,8 @@ func checkGet(req *http.Request, res http.ResponseWriter, statusLogger *statusLo return false } -func decodeRequest(res http.ResponseWriter, req *http.Request, statusLogger *statusLogger) (bool, trustlessutils.Request) { - rootCid, path, err := trustlesshttp.ParseUrlPath(req.URL.Path) +func decodeRequest(res http.ResponseWriter, req *http.Request, unescapedPath string, statusLogger *statusLogger) (bool, trustlessutils.Request) { + rootCid, path, err := trustlesshttp.ParseUrlPath(unescapedPath) if err != nil { if errors.Is(err, trustlesshttp.ErrPathNotFound) { errorResponse(res, statusLogger, http.StatusNotFound, err) @@ -205,8 +211,8 @@ func decodeRequest(res http.ResponseWriter, req *http.Request, statusLogger *sta } } -func decodeRetrievalRequest(cfg HttpServerConfig, res http.ResponseWriter, req *http.Request, statusLogger *statusLogger) (bool, types.RetrievalRequest) { - ok, request := decodeRequest(res, req, statusLogger) +func decodeRetrievalRequest(cfg HttpServerConfig, res http.ResponseWriter, req *http.Request, unescapedPath string, statusLogger *statusLogger) (bool, types.RetrievalRequest) { + ok, request := decodeRequest(res, req, unescapedPath, statusLogger) if !ok { return false, types.RetrievalRequest{} }