From 960d8a5ecb0974756c48fd2e8a810b48e27d2e36 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Thu, 16 Nov 2023 11:54:55 +0300 Subject: [PATCH] unixfs/feather: add a basic test --- cmd/feather/main.go | 11 ++- unixfs/feather/entry.go | 56 +++++++++++- unixfs/feather/feather_test.go | 83 ++++++++++++++++++ .../testdata/file-with-many-raw-leaves.car | Bin 0 -> 5738 bytes 4 files changed, 143 insertions(+), 7 deletions(-) create mode 100644 unixfs/feather/feather_test.go create mode 100644 unixfs/feather/testdata/file-with-many-raw-leaves.car diff --git a/cmd/feather/main.go b/cmd/feather/main.go index 6f4e62cd68..321a1b0e85 100644 --- a/cmd/feather/main.go +++ b/cmd/feather/main.go @@ -38,15 +38,20 @@ Example: %s bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi`, err, os.Args[0], os.Args[0]) } - r, err := feather.DownloadFile(c) + f, err := feather.NewClient(feather.WithStaticGateway("http://localhost:8080/")) if err != nil { - return fmt.Errorf("error starting file download: %w", err) + return fmt.Errorf("creating feather client: %w", err) + } + + r, err := f.DownloadFile(c) + if err != nil { + return fmt.Errorf("starting file download: %w", err) } defer r.Close() _, err = io.Copy(os.Stdout, r) if err != nil { - return fmt.Errorf("error downloading file: %w", err) + return fmt.Errorf("downloading file: %w", err) } return nil } diff --git a/unixfs/feather/entry.go b/unixfs/feather/entry.go index e819454044..c0a943a6f6 100644 --- a/unixfs/feather/entry.go +++ b/unixfs/feather/entry.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "encoding/binary" + "errors" "fmt" "io" "net/http" @@ -36,7 +37,6 @@ func init() { cbor.RegisterCborType(carHeader{}) } -const gateway = "http://localhost:8080/ipfs/" const maxHeaderSize = 32 * 1024 * 1024 // 32MiB const maxBlockSize = 2 * 1024 * 1024 // 2MiB const maxCidSize = 4096 @@ -58,18 +58,66 @@ type downloader struct { readErr error } +type Client struct { + httpClient *http.Client + hostname string +} + +type Option func(*Client) error + +// WithHTTPClient allows to use a [http.Client] of your choice. +func WithHTTPClient(client *http.Client) Option { + return func(c *Client) error { + c.httpClient = client + return nil + } +} + +// WithStaticGateway sets a static gateway which will be used for all requests. +func WithStaticGateway(gateway string) Option { + if len(gateway) != 0 && gateway[len(gateway)-1] == '/' { + gateway = gateway[:len(gateway)-1] + } + gateway += "/ipfs/" + + return func(c *Client) error { + c.hostname = gateway + return nil + } +} + +var ErrNoAvailableDataSource = errors.New("no data source") + +func NewClient(opts ...Option) (*Client, error) { + c := &Client{ + httpClient: http.DefaultClient, + } + + for _, opt := range opts { + if err := opt(c); err != nil { + return nil, err + } + } + + if c.hostname == "" { + return nil, ErrNoAvailableDataSource + } + + return c, nil +} + // DownloadFile takes in a [cid.Cid] and return an [io.ReadCloser] which streams the deserialized file. // You MUST always call the Close method when you are done using it else it would leak resources. -func DownloadFile(c cid.Cid) (io.ReadCloser, error) { +func (client *Client) DownloadFile(c cid.Cid) (io.ReadCloser, error) { c = normalizeCidv0(c) - req, err := http.NewRequest("GET", gateway+c.String()+"?dag-scope=entity", bytes.NewReader(nil)) + req, err := http.NewRequest("GET", client.hostname+c.String()+"?dag-scope=entity", bytes.NewReader(nil)) if err != nil { return nil, err } req.Header.Add("Accept", "application/vnd.ipld.car;dups=y;order=dfs;version=1") - resp, err := http.DefaultClient.Do(req) + resp, err := client.httpClient.Do(req) if err != nil { return nil, err } diff --git a/unixfs/feather/feather_test.go b/unixfs/feather/feather_test.go new file mode 100644 index 0000000000..7359cdc9b7 --- /dev/null +++ b/unixfs/feather/feather_test.go @@ -0,0 +1,83 @@ +package feather_test + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "io" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/exchange/offline" + "github.com/ipfs/boxo/gateway" + "github.com/ipfs/boxo/unixfs/feather" + "github.com/ipfs/go-cid" + carblockstore "github.com/ipld/go-car/v2/blockstore" + "github.com/stretchr/testify/assert" +) + +func newGateway(t *testing.T, fixture string) (*httptest.Server, cid.Cid) { + t.Helper() + + r, err := os.Open(filepath.Join("./testdata", fixture)) + assert.NoError(t, err) + + blockStore, err := carblockstore.NewReadOnly(r, nil) + assert.NoError(t, err) + + t.Cleanup(func() { + blockStore.Close() + r.Close() + }) + + cids, err := blockStore.Roots() + assert.NoError(t, err) + assert.Len(t, cids, 1) + + blockService := blockservice.New(blockStore, offline.Exchange(blockStore)) + + backend, err := gateway.NewBlocksBackend(blockService) + assert.NoError(t, err) + + handler := gateway.NewHandler(gateway.Config{}, backend) + + ts := httptest.NewServer(handler) + t.Cleanup(func() { ts.Close() }) + t.Logf("test server url: %s", ts.URL) + + return ts, cids[0] +} + +func newFeather(t *testing.T, fixture string) (*feather.Client, cid.Cid) { + t.Helper() + + gw, cid := newGateway(t, fixture) + f, err := feather.NewClient(feather.WithHTTPClient(gw.Client()), feather.WithStaticGateway(gw.URL)) + assert.NoError(t, err) + return f, cid +} + +func mustParseHex(s string) []byte { + v, err := hex.DecodeString(s) + if err != nil { + panic(err) + } + return v +} + +func TestFileWithManyRawLeaves(t *testing.T) { + f, root := newFeather(t, "file-with-many-raw-leaves.car") + file, err := f.DownloadFile(root) + assert.NoError(t, err) + defer func() { assert.NoError(t, file.Close()) }() + h := sha256.New() + _, err = io.Copy(h, file) + assert.NoError(t, err) + + if !bytes.Equal(h.Sum(nil), mustParseHex("1cf7caa4f9894d3316fd730598e1ec7cd7d10c3ca87bc09431821f81aa473298")) { + t.Error("decoded content does not match expected") + } +} diff --git a/unixfs/feather/testdata/file-with-many-raw-leaves.car b/unixfs/feather/testdata/file-with-many-raw-leaves.car new file mode 100644 index 0000000000000000000000000000000000000000..ee76a4eedba213445a98cc2996ead80b1d0ebd35 GIT binary patch literal 5738 zcma)=MNk|Hv~6*hgrI@OT^fhruE8a^y9IZb0Kwgz#+wkN8wd^|IKd&%xI^PEf%kvi zdV@QOIyEio2TxYi9mVyCV@Z#38V z)H-e%cP>@)j=G(Xjfba$>u1Cvr2mEq*s$mkbqT2D-@0RK)wuBlBK2Csje~WeB_Zna zu4(c1fKB>4+b$de1kx~!|Al9Q8C6Z3c0AnnGD)gZV>st29(Ol}eaYdN(O!rX?*RXT zf#g5OHc?3|A+iX3C!yl8h{rUTFRcu7oI)<0mhVV(n#nndm_uYG~G-amxEkrdCJ$_*S*|{()Kl zS&(PZhp=Erl7V0s^myNRT?(tbEz4LzOiACBhGQQSZ~i|R6^j@HiS!FIbr{CK1{H+y ze~Bn$%uwep9pa_2K9L~9Kohm?;Xab-_Fjs=2&ol`wpk?vxb#@*O$UG6OWQtZPC6o; zwBPX4*t;{>6^psJza4I9m_FnOu%>n<2MQ!$EJvZW_h{Y?R6AWhpddQi=Z6_D&KK>^ zk6eX3hXtxS6WDD!8zf|%- zs7CEwH<%}L1@=+k4cW=^gYNk9I>J>K6=p&MMO_@-9sA@*lA0dgI33rk2yff>ePp8M zh*ep{?+$NnPmUKPFu?|!T~Uemo>yVAt|jQ~_uvqBdSh-ulx)@bnV+g6g8LNK*KKjC zIIgqffMs=K8y5%2;Da@Hj(nDY>4)CC@xxo&#?;EA-kp;^>DP?~jL?pBNqW(j5AHs- zd01uo4a0*N7R~5>gu1J=Z1KJB2ImyT=pR8y0@*WBDOr-mf69_V&=ocWHxq_O+epo| zgYyH;yIArI-Vj{hdUYcAOPJaX-((dpc069-olBZT4}P=#h(tnzy)BZZ{ICfUxu%vR zy`nVm7)J-i>RTl0miKS@$i&wJG87Rs&KV0v8Y~G5WNdElg9_Xorf8UXg@XjG|EzFh zc&Zu$DTZ{htn&xX*+eTV4-~kcRgilVRnp}NR!+NL!!YBDs}>}jvRCPWXpi}V0q<1CdeugD{^WaqVq9I#U{y!w+T__3m{KF;*x2A0;H%tU6rr)DL7$1-Dnt&TR$ zCatVzm4V&})3?Y!#?cdA7KL^Jt@a4r>K*FtSGg$NWpQ(kRn4Ub?&pY;B`7SGlR4oZ z_<0Np&JyYXo`F3YF2ZOhHNPEYqgDv_ z6l6>n0lp%U1pjT5X%NQ)$y;xUuo>N|+ieTH86l0Wr6$;m#K)#%X8YGnTwF=#Uj+R(yoe5*>OXSKos%@jxxZqWOLjVfca zOxuLvE(mP}xk@}_BD{I`6y8>em}1Jq&1%D>cNh<|#UB(A5d(}pHmRYr$B?ntQ^Jro znLuc)NHzf;QQLU)nk>ui;i+{9!#c#&Zhk!!RRlRP3wWSR4!J|p?t-y{dY#buO|fj2 zhh;DV;W9yvid`6>`Y5@1uytB~c<_^ETG`TjCR1ZGY~fbx!tEK77$t@v`|$d6cfDzu zV0Wm}Sz^Jbyb|t#b6tInRGzK?QPf=ZefS3XSN@TYm%dfhBL7(8zuf4K^2T5IIaxU> zRY`!8x;8nOBk0@Qa*@Sr)1r&#kK&KxuRdI?8fKDd2EiKgn-2TDf5lEL?Du3oJl@CF z2}|@TZBOUU&qP)IDt(c2tJ-&TIQK~)34qhJ6&@CBsy$5fRn_${fB$yqCXZEu>^p73 z7q33SQHOtqfxzQwT1vVIiXx+hX(NM(a#rPv>Q6ibVp9zV+sMZApwrM>0>79~ZE=Qu zmRMH@rK(?qlMFYaTX;||yG;Uvx>ZFycnn?-fXrvO!o^}ue=Ze+nM)OwSElshj{@CC zEYNvih|2GfIBtWg_PY$q#luXA7+X*%-s*jOWf;SiH9N-%_>rEn-u^_T6u%Z%tEq~I zUcezb7amcVQ8~kIYGd0}-blmHV5m2?u}oIAv~a~Rt6}bOF71s*K|fQQg(fjfIYIQk zWo!5V(QPSh>yzICl!ZzoLvW>#V?La0Wmm`!a;-d~EKCtx+lQwdnsik2b;<+TK_{o7HYuZ8fLP`L}$zY`1RPIPvfDOAU{c4xl4H1 z6})08Z;d-+kp~pX%sERB$sL!03T;rk{jh%%E75uyp~%y##$i~o!ub317{%{E9sc;S zG~g-nkm^lpl|V4~KGZSz+7%YH{#~DLNCL`{x}99tlIAS?t>;E*StP>n-Ifxk_}YC6 zW6o)<);>NKe)@sAtrP=i4qCZ+LjcLok5$SC7?V zGb&kpJ*?o0DO2#a&?u2m~QHQ;Tmji*Y0sqf#(b85& zGMyl9xbR>04-JY{s*L(yA-+%^2}4L);%miNf9p2SE4Qc7DPn;L8g-LKa1B=N*#ue>2!ViMnqa^S60#ben?oyebof}IRY%k4_o z%VZEyR5!-})2B&7^W1%fi}een_xyedUun~1Vy_Ut0s0aIFufn)%3I=~M-%LsF`X^u zx=%j$D0JEEk_(-VSC1x<#fO}I&a9=xjF`A@0_x&iJWKJjs7rqoz9N?Sqd)3~b<7BP zty`QKAD)RtZEwt}+T;gdo`&iRb#@#w7Z$f3{9}p#I*P8cgPY{0a5ZiAGD3>PTNAWY zc&6cbUpCqUiaaN>i}+W~E*_{6Mo#$)(tQ|i_KOvvI6Vycxf$AMD7M|hFdpKGs<^)p^v(yY6> z=17*k2*Ni9&~u&r!@g=DHKDyaF*)xp+ zNuHL(*ML5Dfy7@^b63<{JC1wuYQE>g-i0c4x0Z2E6#6~U8_n)U?X?_xYME4<5R*8h za&)FbJiFqk+9PoE_!g}PCPOKgMelo3=T48gqYKK-V?E$sK45TWW9)!?s4)VWXQmzS z8CbnS5q6qST^6UuywqgbVc16U%1IC4WbsNOzh#9O$!LzP)aL-gYwM!&hr;$V($CXTvOXs&S+asrcKiQ?J)J zJKDW}NdJ23!l&CBTN?8`tW}49*ZsW6@^<8VY?%BJI zWy&I)^X!o(^h2L8A*DRey~T~?025ZOmtPrbNU}Fku$rn@v6X?T-!gAZW~O?Bg&nkpqUPy+opaZSR=k?H?l6Dw*)db==3VF1M-QlZM&sSqKggU?B<;7 z0asXN+y&l8_Ld5c6Mj+1qzC2-6NpwSgu-cYXw8}%m{anPFt_=M_|zpljjk-1PDrKT zh`a5!@K^5FFB#U^FJa7N!_!hj$a%OrXr1P7Ls`pNK?3;!t$0b@@i=?r-s*W z2LK1t0Po?-0-sG>#H@+0i7E>PoJ!G5$)8#&i)80@6*xbp&8-mvT2t!gz9B9=vdamDQFjvj#iU=o8hluT-y!kcM>;ehkLk$=WG{tBmqQ4_M4@w zV#jVDB<-o@>M820-X~ELeQbKsKYq!)!N4Hs(&y6l_>Hif7Fkq5r>7G^dsbm9E#s>; zKv%^!(UiO3K|T>TFpuE)*z4I+g(CVnh{+^8+XVAXG7bBnBIZWQq$n-Vk3WfyUxQiJ z)Q%=N`VPz?VdL0}%BN$b-$~j|dQqJ8vYv>$$oILX!Mn0*B;L%gJV;!(yRi|S1eMF# ziuNm(R7DC-qFP|7$U-DoxcTPP8=?7@vHEAK8oJ;xTC(K&(*_CqiFm0ggT47a1MlYM z-qmI)JGDE0$yz(=Gs5Q!$zWw+fO>3`lZO}q^HAbxV7YWaV6s%v^=G^98`GOw&*^cfB|GE0n5wtxp-)DdOoW*5&Jw5e``TcxTkR{a-Lkw(81Q z`ZNHw`F5tSGhG9h{+ie8I2M3UrieL6l?Ay86ST;R#eXY}!4-6g#97*4C5dmnqM^DN z%qv4D-=LMu3=!=V980+&TP7$mM0vL0Ts?J3Z7!8gHl~ zJcDjC9Op7OTc_lgLYMN{vANCklcGv`%|McX(NMNdapBRanP%CRR&8}#d_j1Y&-iwBw z4l(r`GsNKX-riRB5SjW9yS|U`Far`414b>GtGkn>y9I@$YBIF$!}fgXpq!5`3i8X6 zlt(HslDu8eeD06#UID^Yt%o9O#kbh3zC_u}k>lIzxegR*qA)pA(urpc|C|Ca!jgow zX+h0uDo3^B>?S?BATfr56|=K!b(#`OwM zLI${(r8;ZL*|ah9EK$`&*$v|4vRwW_0{eSX%k21G2&vdFXja;+g5qnf3fl#0%C6xN zfBVx*GcTl2tg|+mYQ*7lpIyo1O+JP7t^qmg05uSMX<80_n zXvnoQ7)<()CH|`hp?_299=jQF$6XetWCJ%$mGM`(kWICXw@g&ZQ!Qbhf9maiWCq*W zsUDUS$zkr}wuUzDYr<-nR9(WnQ(Jobw9TCvJRA%#{Zs6HsOjyix}$Z%(2@3JfaVnE z3IkgZV#OBN3^qc$MF@w}rqIM%(RbII3m3)aG%k!srqaNEmtD#oLe}qbSC`Sl*SMxp z3N78^6J`dRX4wcwz`|#tu@tq4G#EM-bR8uaO-%EK8smLG!L5_3qu(JaV zmI0KpaejxQbTqJUyl(dGdpS#($=V}dg_czZt&gc3?I!0Bgj_L!k1?n>e8Z=gyWpa} z2=2<%ly_f+Of2f?bW{#XI4yUD$+iTk#AWg#qk4GVtTPcDg1WYY_0D{1E%j&4<@Eu& zYk1ys*wRb>LDCAad389A83So<3(A5X>Z4@Bu&e<_<-3ziZjHdkXNl*XfIso^fszU;6pe zxqIXV!)d`6kYp{5V3laJxIy`d3ahZPS&M|*~qrUnruciF4vb+J+F>eCsvPeGPBD5U_i=8+(?A$k-ojZ z+sUfwA&w>sMMn<}iBhlGf$lDvyu=YjcSDi5rmy0va@M}lU4`g_s`xM*xT^Y`koJIDBhj*g$vJ` zMMkke7LT|VeK=Cu3mb@c`Oc1 z*2e)txvT=+NuB($)=w^M_~_JR&b(m