cohost2json: retry liked chosts page scraping logic (#33)

delan · Dec 30, 2024 · 37322fd · 37322fd
1 parent b0618ea
commit 37322fd
Showing 1 changed file with 21 additions and 26 deletions.
diff --git a/src/command/cohost2json.rs b/src/command/cohost2json.rs
@@ -138,24 +138,27 @@ pub async fn main(args: Cohost2json) -> eyre::Result<()> {
                     liked_page * 20
                 );
 
-                let response = get_text(&client, &url).await?;
-                let document = Html::parse_document(&response);
-
-                let selector = Selector::parse("script#__COHOST_LOADER_STATE__")
-                    .expect("guaranteed by argument");
-                let node = document
-                    .select(&selector)
-                    .next()
-                    .ok_or_eyre("failed to find script#__COHOST_LOADER_STATE__")?;
-                let texts = node.text().collect::<Vec<_>>();
-                let (text, rest) = texts
-                    .split_first()
-                    .ok_or_eyre("script element has no text nodes")?;
-                if !rest.is_empty() {
-                    error!("script element has more than one text node");
-                }
-
-                let liked_store = serde_json::from_str::<LikedPostsState>(text)?.liked_posts_feed;
+                let liked_store = get_with_retries(&client, &url, |body| {
+                    let body = str::from_utf8(&body)?;
+                    let document = Html::parse_document(body);
+                    let selector = Selector::parse("script#__COHOST_LOADER_STATE__")
+                        .expect("guaranteed by argument");
+                    let node = document
+                        .select(&selector)
+                        .next()
+                        .ok_or_eyre("failed to find script#__COHOST_LOADER_STATE__")?;
+                    let texts = node.text().collect::<Vec<_>>();
+                    let (text, rest) = texts
+                        .split_first()
+                        .ok_or_eyre("script element has no text nodes")?;
+                    if !rest.is_empty() {
+                        error!("script element has more than one text node");
+                    }
+                    let liked_store =
+                        serde_json::from_str::<LikedPostsState>(text)?.liked_posts_feed;
+                    Ok(liked_store)
+                })
+                .await?;
 
                 if !liked_store.paginationMode.morePagesForward {
                     break;
@@ -174,10 +177,6 @@ pub async fn main(args: Cohost2json) -> eyre::Result<()> {
     Ok(())
 }
 
-async fn get_text(client: &Client, url: &str) -> eyre::Result<String> {
-    get_with_retries(client, url, text).await
-}
-
 async fn get_json<T: DeserializeOwned>(client: &Client, url: &str) -> eyre::Result<T> {
     get_with_retries(client, url, json).await
 }
@@ -238,10 +237,6 @@ async fn get_response_once(client: &Client, url: &str) -> reqwest::Result<Respon
     client.get(url).send().await
 }
 
-fn text(body: Bytes) -> eyre::Result<String> {
-    Ok(str::from_utf8(&body)?.to_owned())
-}
-
 fn json<T: DeserializeOwned>(body: Bytes) -> eyre::Result<T> {
     Ok(serde_json::from_slice(&body)?)
 }