From 89e930fe0cbeaf0b2c4de47a047fe6b337142cdd Mon Sep 17 00:00:00 2001 From: 0xZensh Date: Sat, 2 Dec 2023 20:03:29 +0800 Subject: [PATCH] chore: improve collection SSR --- dist/app.js | 1 + dist/ssr.js | 82 +++++++++++++++++++++++++++++--------------- package.json | 2 +- src/app.ts | 1 + src/ssr.ts | 96 ++++++++++++++++++++++++++++++++++------------------ 5 files changed, 122 insertions(+), 60 deletions(-) diff --git a/dist/app.js b/dist/app.js index 7c6f54c..1367e25 100644 --- a/dist/app.js +++ b/dist/app.js @@ -21,6 +21,7 @@ export async function initApp(app) { router.get('/pub/:cid', renderPublication); router.get('/group/:gid', renderGroup); router.get('/group/:gid/collection', renderCollection); + router.get('/group/:gid/:other', renderGroup); router.all('/:other+', (ctx) => { ctx.redirect('/'); ctx.status = 307; diff --git a/dist/ssr.js b/dist/ssr.js index 7c082be..6d8cf25 100644 --- a/dist/ssr.js +++ b/dist/ssr.js @@ -144,6 +144,9 @@ export async function renderCollection(ctx) { $('title').text(siteInfo.title); $('meta[name="description"]').prop('content', siteInfo.desc); } + if (!_cid || Array.isArray(_cid)) { + return renderGroup(ctx); + } try { const doc = await getCollection(headers, _cid); const [language, info] = getCollectionInfo(doc, lang) ?? []; @@ -453,40 +456,65 @@ async function listCollections(headers, gid) { const api = new URL('/v1/collection/list', writingBase); headers.accept = 'application/cbor'; headers['content-type'] = 'application/cbor'; - const res = await fetch(api, { - method: 'POST', - headers, - body: Buffer.from(encode({ - gid: gid.toBytes(), - status: 2, - fields: ['info', 'updated_at'], - })), - }); - if (res.status !== 200) { - throw createError(res.status, await res.text()); + const output = new Array(); + const input = { + gid: gid.toBytes(), + page_size: 100, + status: 2, + fields: ['info', 'updated_at'], + page_token: undefined, + }; + let i = 7; + while (i > 0) { + i -= 1; + const res = await fetch(api, { + method: 'POST', + headers, + body: Buffer.from(encode(input)), + }); + if (res.status !== 200) { + break; + } + const data = await res.arrayBuffer(); + const obj = decode(Buffer.from(data)); + output.push(...obj.result); + if (!obj.next_page_token) { + break; + } + input.page_token = obj.next_page_token; } - const data = await res.arrayBuffer(); - const obj = decode(Buffer.from(data)); - return obj.result; + return output; } async function listLatestCollections(headers) { const api = new URL('/v1/collection/list_latest', writingBase); headers.accept = 'application/cbor'; headers['content-type'] = 'application/cbor'; - const res = await fetch(api, { - method: 'POST', - headers, - body: Buffer.from(encode({ - page_size: 100, - fields: ['info', 'updated_at'], - })), - }); - if (res.status !== 200) { - throw createError(res.status, await res.text()); + const output = new Array(); + const input = { + page_size: 100, + fields: ['info', 'updated_at'], + page_token: undefined, + }; + let i = 7; + while (i > 0) { + i -= 1; + const res = await fetch(api, { + method: 'POST', + headers, + body: Buffer.from(encode(input)), + }); + if (res.status !== 200) { + break; + } + const data = await res.arrayBuffer(); + const obj = decode(Buffer.from(data)); + output.push(...obj.result); + if (!obj.next_page_token) { + break; + } + input.page_token = obj.next_page_token; } - const data = await res.arrayBuffer(); - const obj = decode(Buffer.from(data)); - return obj.result; + return output; } async function listCollectionChildren(headers, id) { const api = new URL('/v1/collection/list_children', writingBase); diff --git a/package.json b/package.json index 2f6acee..eaa3638 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "webscraper", - "version": "1.2.1", + "version": "1.2.2", "description": "", "private": true, "main": "dist/main.js", diff --git a/src/app.ts b/src/app.ts index d110489..4821711 100644 --- a/src/app.ts +++ b/src/app.ts @@ -38,6 +38,7 @@ export async function initApp(app: Koa): Promise { router.get('/pub/:cid', renderPublication) router.get('/group/:gid', renderGroup) router.get('/group/:gid/collection', renderCollection) + router.get('/group/:gid/:other', renderGroup) router.all('/:other+', (ctx) => { ctx.redirect('/') ctx.status = 307 diff --git a/src/ssr.ts b/src/ssr.ts index 115292b..93c8ae5 100644 --- a/src/ssr.ts +++ b/src/ssr.ts @@ -180,8 +180,12 @@ export async function renderCollection(ctx: Context): Promise { $('meta[name="description"]').prop('content', siteInfo.desc) } + if (!_cid || Array.isArray(_cid)) { + return renderGroup(ctx) + } + try { - const doc = await getCollection(headers, _cid as string) + const doc = await getCollection(headers, _cid) const [language, info] = getCollectionInfo(doc, lang) ?? [] if (!info || !language) { throw createError(404, 'collection not found') @@ -690,25 +694,39 @@ async function listCollections( const api = new URL('/v1/collection/list', writingBase) headers.accept = 'application/cbor' headers['content-type'] = 'application/cbor' - const res = await fetch(api, { - method: 'POST', - headers, - body: Buffer.from( - encode({ - gid: gid.toBytes(), - status: 2, - fields: ['info', 'updated_at'], - }) - ), - }) + const output = new Array() + + const input = { + gid: gid.toBytes(), + page_size: 100, + status: 2, + fields: ['info', 'updated_at'], + page_token: undefined, + } - if (res.status !== 200) { - throw createError(res.status, await res.text()) + let i = 7 + while (i > 0) { + i -= 1 + const res = await fetch(api, { + method: 'POST', + headers, + body: Buffer.from(encode(input)), + }) + + if (res.status !== 200) { + break + } + + const data = await res.arrayBuffer() + const obj = decode(Buffer.from(data)) + output.push(...obj.result) + if (!obj.next_page_token) { + break + } + input.page_token = obj.next_page_token } - const data = await res.arrayBuffer() - const obj = decode(Buffer.from(data)) - return obj.result + return output } async function listLatestCollections( @@ -717,24 +735,38 @@ async function listLatestCollections( const api = new URL('/v1/collection/list_latest', writingBase) headers.accept = 'application/cbor' headers['content-type'] = 'application/cbor' - const res = await fetch(api, { - method: 'POST', - headers, - body: Buffer.from( - encode({ - page_size: 100, - fields: ['info', 'updated_at'], - }) - ), - }) + const output = new Array() - if (res.status !== 200) { - throw createError(res.status, await res.text()) + const input = { + page_size: 100, + fields: ['info', 'updated_at'], + page_token: undefined, } - const data = await res.arrayBuffer() - const obj = decode(Buffer.from(data)) - return obj.result + let i = 7 + while (i > 0) { + i -= 1 + + const res = await fetch(api, { + method: 'POST', + headers, + body: Buffer.from(encode(input)), + }) + + if (res.status !== 200) { + break + } + + const data = await res.arrayBuffer() + const obj = decode(Buffer.from(data)) + output.push(...obj.result) + if (!obj.next_page_token) { + break + } + input.page_token = obj.next_page_token + } + + return output } async function listCollectionChildren(