From 55483c174b066b488b07132f2fa607429fcc6592 Mon Sep 17 00:00:00 2001 From: tevko Date: Wed, 5 Feb 2025 09:22:46 -0600 Subject: [PATCH] better json streaming and filtering --- client-report/src/components/app.jsx | 16 +++++++++++----- .../src/components/lists/consensusNarrative.jsx | 2 -- .../src/components/lists/groupsNarrative.jsx | 2 -- .../src/components/lists/topicNarrative.jsx | 2 -- .../components/lists/uncertaintyNarrative.jsx | 2 -- server/src/routes/reportNarrative.ts | 6 +++--- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/client-report/src/components/app.jsx b/client-report/src/components/app.jsx index c44a7aa69..05ce830ea 100644 --- a/client-report/src/components/app.jsx +++ b/client-report/src/components/app.jsx @@ -198,11 +198,17 @@ const App = (props) => { const decodedChunk = decoder.decode(value, { stream: true }); if (!decodedChunk.includes("POLIS-PING:")) { - const c = JSON.parse(decodedChunk); - setNarrative((prevNarrative) => ({ - ...(prevNarrative || {}), - ...c, - })); + decodedChunk.split(`|||`).filter(Boolean).forEach((j) => { + try { + const c = JSON.parse(j); + setNarrative((prevNarrative) => ({ + ...(prevNarrative || {}), + ...c, + })) + } catch (error) { + console.log(error, j) + } + }); } } }; diff --git a/client-report/src/components/lists/consensusNarrative.jsx b/client-report/src/components/lists/consensusNarrative.jsx index c76720dbd..fb2317ea6 100644 --- a/client-report/src/components/lists/consensusNarrative.jsx +++ b/client-report/src/components/lists/consensusNarrative.jsx @@ -15,10 +15,8 @@ const ConsensusNarrative = ({ try { const txt = model === "claude" ? narrative.responseClaude.content[0].text : narrative.responseGemini; - console.log("Raw narrative text:", txt); // Log raw text const narrativeJSON = model === "claude" ? JSON.parse(`{${txt}`) : JSON.parse(txt); - console.log("Parsed narrative JSON:", narrativeJSON); // Log parsed JSON // Extract all citation IDs from the narrative structure const uniqueTids = narrativeJSON.paragraphs.reduce((acc, paragraph) => { diff --git a/client-report/src/components/lists/groupsNarrative.jsx b/client-report/src/components/lists/groupsNarrative.jsx index 97da12064..e604d34b7 100644 --- a/client-report/src/components/lists/groupsNarrative.jsx +++ b/client-report/src/components/lists/groupsNarrative.jsx @@ -18,10 +18,8 @@ const GroupsNarrative = ({ try { const txt = model === "claude" ? narrative?.responseClaude.content[0].text : narrative?.responseGemini; - console.log("Raw narrative text:", txt); // Log raw text const narrativeJSON = model === "claude" ? JSON.parse(`{${txt}`) : JSON.parse(txt); - console.log("Parsed narrative JSON:", narrativeJSON); // Log parsed JSON // Extract all citation IDs from the narrative structure const uniqueTids = narrativeJSON.paragraphs.reduce((acc, paragraph) => { diff --git a/client-report/src/components/lists/topicNarrative.jsx b/client-report/src/components/lists/topicNarrative.jsx index a4b358fad..6919b367b 100644 --- a/client-report/src/components/lists/topicNarrative.jsx +++ b/client-report/src/components/lists/topicNarrative.jsx @@ -17,10 +17,8 @@ const TopicNarrative = ({ try { const txt = model === "claude" ? narrative?.responseClaude.content[0].text : narrative?.responseGemini; - console.log(`Raw narrative text for topic ${topicName}:`, txt); const narrativeJSON = model === "claude" ? JSON.parse(`{${txt}`) : JSON.parse(txt); - console.log(`Parsed narrative JSON for topic ${topicName}:`, narrativeJSON); // Extract all citation IDs from the narrative structure const uniqueTids = narrativeJSON.paragraphs.reduce((acc, paragraph) => { diff --git a/client-report/src/components/lists/uncertaintyNarrative.jsx b/client-report/src/components/lists/uncertaintyNarrative.jsx index a102a1af0..a7221e4d7 100644 --- a/client-report/src/components/lists/uncertaintyNarrative.jsx +++ b/client-report/src/components/lists/uncertaintyNarrative.jsx @@ -18,10 +18,8 @@ const UncertaintyNarrative = ({ try { const txt = model === "claude" ? narrative?.responseClaude.content[0].text : narrative?.responseGemini; - console.log("Raw narrative text:", txt); // Log raw text const narrativeJSON = model === "claude" ? JSON.parse(`{${txt}`) : JSON.parse(txt); - console.log("Parsed narrative JSON:", narrativeJSON); // Log parsed JSON // Extract all citation IDs from the narrative structure const uniqueTids = narrativeJSON.paragraphs.reduce((acc, paragraph) => { diff --git a/server/src/routes/reportNarrative.ts b/server/src/routes/reportNarrative.ts index 26c97329b..63f37795b 100644 --- a/server/src/routes/reportNarrative.ts +++ b/server/src/routes/reportNarrative.ts @@ -314,7 +314,7 @@ export async function handle_GET_reportNarrative( ? "NO_CONTENT_AFTER_FILTER" : undefined, }, - }) + }) + `|||` ); } else { if ( @@ -370,7 +370,7 @@ export async function handle_GET_reportNarrative( ? "NO_CONTENT_AFTER_FILTER" : undefined, }, - }) + }) + `|||` ); } else { const responseClaude = await anthropic.messages.create({ @@ -441,7 +441,7 @@ export async function handle_GET_reportNarrative( ? "NO_CONTENT_AFTER_FILTER" : undefined, }, - }) + }) + `|||` ); } }