re-add generator

.gitignore

@@ -10,11 +10,4 @@ yarn-error.log
 
 .github
 
-# Index build artifact
-/_site
-
-# generator
-site_generator/site/assets/fuse-index.json
-site_generator/site/assets/scrapers.json
-
 scrapers/JSON_info/config.py

site_generator/README.md

@@ -0,0 +1,10 @@
+# Dyanmically Generated CommunityScrapers index
+
+## site dependencies
+- `ago.js`
+  - https://github.com/sebastiansandqvist/s-ago
+  - modified to remove future dates
+- `fuse-index.json`
+  - pre-generated fuse index for faster searching
+- `scrapers.json`
+  - generated list of scrapers

site_generator/site/assets/ago.js

@@ -0,0 +1,21 @@
+// s-ago https://github.com/sebastiansandqvist/s-ago
+// LICENCE MIT by Sebastian Sandqvist
+
+function formatDate(diff, divisor, unit, past) {
+  var val = Math.round(Math.abs(diff) / divisor);
+  return val <= 1 ? past : `${val} ${unit}s ago`;
+}
+var units = [
+  { max: 518400000, value: 86400000, name: "day", past: "yesterday" },
+  { max: 2419200000, value: 604800000, name: "week", past: "last week" },
+  { max: 28512000000, value: 2592000000, name: "month", past: "last month" }, // max: 11 months
+];
+export default function ago(date, max) {
+  var diff = Date.now() - date.getTime();
+  for (const unit of units) {
+    if (Math.abs(diff) < unit.max || (max && unit.name === max)) {
+      return formatDate(diff, unit.value, unit.name, unit.past);
+    }
+  }
+  return formatDate(diff, 31536000000, "year", "last year");
+}

site_generator/site/assets/index.css

@@ -0,0 +1,62 @@
+.tooltip span {
+  display: none;
+  color: #eee;
+  text-decoration: none;
+  padding: 3px;
+}
+.tooltip:hover span {
+  display: block;
+  position: absolute;
+  background-color: #000;
+  border: 1px solid #ccc;
+  margin: 2px 10px;
+  margin-left: -25px;
+}
+table {
+  width: 100%;
+}
+thead {
+  position: sticky;
+  top: 0;
+  background: var(--bulma-scheme-main) !important;
+}
+.navbar {
+  text-align: center;
+  display: block;
+}
+.navbar > .navbar-item {
+  display: inline-block;
+}
+.pre {
+  -webkit-overflow-scrolling: touch;
+  -moz-osx-font-smoothing: auto;
+  -webkit-font-smoothing: auto;
+  background-color: var(--bulma-pre-background);
+  color: var(--bulma-pre);
+  font-size: var(--bulma-pre-font-size);
+  overflow-x: auto;
+  padding: var(--bulma-pre-padding);
+  white-space: pre;
+  word-wrap: normal;
+  font-family: var(--bulma-code-family);
+  margin: 0;
+  display: block;
+  unicode-bidi: isolate;
+}
+details {
+  word-wrap: normal;
+  white-space: pre;
+}
+mark {
+  color: #000
+}
+#top-btn {
+  position: fixed;
+  bottom: 20px;
+  right: 20px;
+  border: none;
+  border-radius: 50%;
+  padding: 10px;
+  cursor: pointer;
+  z-index: 1000;
+}

site_generator/site/assets/scraperlist.js

@@ -0,0 +1,178 @@
+import Fuse from "https://cdn.jsdelivr.net/npm/[email protected]/dist/fuse.basic.min.mjs";
+import ago from "./ago.js";
+// constant elements
+const searchInput = document.querySelector("#search");
+const table = document.getElementById("scraper-list");
+const searchTypes = {
+  name: { tooltip: "Search:", emoji: "🔍" },
+  url: { tooltip: "URL:", emoji: "🔗" },
+  fragment: { tooltip: "Smart:", emoji: "🧠" },
+};
+
+// helper functions
+const emojiBool = (value) => (value ? "✅" : "❌");
+const noMatch = "—";
+const returnMatch = (bool, value) => (bool ? value : noMatch);
+const anyTrue = (obj) => Object.values(obj).some((v) => v);
+// https://stackoverflow.com/a/54265129
+function debounce(f, interval) {
+  let timer = null;
+  return (...args) => {
+    clearTimeout(timer);
+    return new Promise((resolve) => {
+      timer = setTimeout(() => resolve(f(...args)), interval);
+    });
+  };
+}
+
+// tooltip helper
+const createToolTip = (target, stypeObj) => {
+  // if false, don't add tooltip
+  if (!anyTrue(stypeObj)) return (target.textContent = noMatch);
+  // generate tooltip text dynamically
+  let tooltipArray = [];
+  let typeText = "";
+  // add all applicable tooltips
+  for (const [key, value] of Object.entries(searchTypes)) {
+    const check = stypeObj[key];
+    if (check !== undefined) {
+      tooltipArray.push(`${value.tooltip} ${emojiBool(check)}`);
+      if (check) typeText += value.emoji;
+    }
+  }
+  // create tooltip text
+  const tooltipElement = document.createElement("span");
+  tooltipElement.textContent = tooltipArray.join(" | ");
+  target.textContent = typeText;
+  target.classList.add("tooltip");
+  target.appendChild(tooltipElement);
+};
+
+const createDetails = (values, fallback, searchValue, expand = false) => {
+  const preContainer = document.createElement("div");
+  if (!values?.length) preContainer.textContent = fallback;
+  else {
+    // search
+    values = values.map((value) =>
+      searchValue && value.toLowerCase().includes(searchValue.toLowerCase())
+        ? `<mark>${value}</mark>`
+        : value,
+    );
+    const summary = document.createElement("summary");
+    summary.textContent = fallback;
+    const p = document.createElement("p");
+    p.innerHTML = values.join("\n");
+    const detailsBox = document.createElement("details");
+    detailsBox.appendChild(p);
+    detailsBox.appendChild(summary);
+    preContainer.appendChild(detailsBox);
+    if (expand && searchValue) detailsBox.open = true;
+  }
+  return preContainer;
+};
+
+const siteDetails = (sites, searchValue, expand = false) => {
+  const preContainer = createDetails(
+    sites.slice(1),
+    sites[0],
+    searchValue,
+    expand,
+  );
+  preContainer.classList.add("pre");
+  return preContainer;
+};
+
+const scrapes = (name, scrapes, searchValue, expand = false) =>
+  createDetails(scrapes, name, searchValue, expand);
+
+const setTable = (scrapers, searchValue = "") => {
+  if (table.rows.length) table.innerHTML = "";
+  scrapers.forEach((scp, idx) => {
+    const sType = scp.searchTypes;
+    const row = table.insertRow();
+    // name
+    row
+      .insertCell(0)
+      .appendChild(scrapes(scp.name, scp.scrapes, searchValue, idx <= 5));
+    // supported sites
+    row
+      .insertCell(1)
+      .appendChild(siteDetails(scp.sites, searchValue, idx <= 5));
+    // scene scraping
+    createToolTip(row.insertCell(2), sType.scene);
+    // gallery scraping
+    createToolTip(row.insertCell(3), sType.gallery);
+    // movie scraping
+    row.insertCell(4).textContent = returnMatch(sType.movie.url, "🔗");
+    // performer scraping
+    createToolTip(row.insertCell(5), sType.performer);
+    // requires
+    row.insertCell(6).textContent = returnMatch(scp.requires.python, "🐍");
+    row.insertCell(7).textContent = returnMatch(scp.requires.cdp, "🌐");
+    row.insertCell(8).textContent = ago(new Date(scp.lastUpdate));
+  });
+};
+
+const keys = [
+  {
+    name: "filename",
+    weight: 2,
+  },
+  {
+    name: "name",
+    weight: 20,
+  },
+  {
+    name: "sites",
+    weight: 2,
+  },
+  {
+    name: "scrapes",
+    weight: 10,
+  },
+  {
+    name: "hosts",
+    weight: 10,
+  },
+];
+
+// fuse config
+const fuseConfig = {
+  keys,
+  threshold: 0.4,
+  shouldSort: true,
+  includeScore: true, // debugging
+  minMatchCharLength: 3,
+};
+
+// init fuse
+let fuse;
+
+// fuse search
+async function search(searchValue) {
+  if (searchValue.length < 3) return setTable(rawScraperList);
+  const results = fuse.search(searchValue, {
+    limit: 20,
+  });
+  console.debug(searchValue, results);
+  const filterTable = results.map((result) => result.item);
+  setTable(filterTable, searchValue);
+  window.location.hash = searchValue
+}
+
+// parse scrapers.json
+const rawScraperList = await fetch("assets/scrapers.json").then((response) =>
+  response.json(),
+);
+setTable(rawScraperList);
+const fuseIndex = await fetch("assets/fuse-index.json")
+  .then((response) => response.json())
+  .then((data) => Fuse.parseIndex(data));
+fuse = new Fuse(rawScraperList, fuseConfig, fuseIndex);
+// if query in URL, jump automatically
+const query = window.location.hash.slice(1)
+if (query) {
+  searchInput.value = query;
+  search(query);
+}
+searchInput.addEventListener("input", event => debounce(search(event.target.value), 300));

site_generator/site/index.html

@@ -0,0 +1,64 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Stash Scrapers List</title>
+    <link
+      rel="stylesheet"
+      href="https://cdn.jsdelivr.net/npm/[email protected]/css/bulma.min.css"
+    />
+    <link rel="stylesheet" href="assets/index.css" />
+  </head>
+  <body>
+    <nav class="navbar" role="navigation">
+      <a
+        class="navbar-item"
+        href="https://github.com/stashapp/CommunityScrapers"
+        >GitHub</a
+      >
+      <a
+        class="navbar-item"
+        href="https://docs.stashapp.cc/beginner-guides/guide-to-scraping/"
+        >Guide to Scraping</a
+      >
+      <a
+        class="navbar-item"
+        href="https://docs.stashapp.cc/in-app-manual/scraping/"
+        >Scraping Help</a
+      >
+    </nav>
+    <section class="section">
+      <input type="text" class="input" id="search" placeholder="Search..." />
+      <hr />
+      <table
+        class="table is-fullwidth is-striped is-hoverable"
+        id="scraper-table"
+      >
+        <thead>
+          <tr>
+            <th id="scraper_name">Name</th>
+            <th id="scraper_sites">Sites</th>
+            <th id="scraper_scene">Scene</th>
+            <th id="scraper_gallery">Gallery</th>
+            <th id="scraper_movie">Movie</th>
+            <th id="scraper_performer">Performer</th>
+            <th id="scraper_python">Python</th>
+            <th id="scraper_cdp">CDP</th>
+            <th id="lastupdate">Last Update</th>
+          </tr>
+        </thead>
+        <tbody id="scraper-list"></tbody>
+      </table>
+    </section>
+    <button
+      id="top-btn"
+      class="has-background-link"
+      onclick="window.scrollTo({ top: 0, behavior: 'smooth' })"
+      id="scrollToTopBtn"
+    >
+      ⬆️
+    </button>
+    <script type="module" src="assets/scraperlist.js"></script>
+  </body>
+</html>

site_generator/src/fuse-index.js

@@ -0,0 +1,37 @@
+const Fuse = require("fuse.js");
+const fs = require("fs");
+
+const data = require("../site/assets/scrapers.json");
+// filename - contains path, not super helpful
+// name - primary key
+// sites - scraping URLs, mostly for exact matching
+// scrapes - manual comment for sites supported
+// hosts - sites but reduced to URL hosts (reduce false positives)
+const keys = [
+  {
+    name: "filename",
+    weight: 2,
+  },
+  {
+    name: "name",
+    weight: 20,
+  },
+  {
+    name: "sites",
+    weight: 2,
+  },
+  {
+    name: "scrapes",
+    weight: 10,
+  },
+  {
+    name: "hosts",
+    weight: 10,
+  },
+];
+const fuseIndex = Fuse.createIndex(keys, data);
+
+fs.writeFileSync(
+  "site/assets/fuse-index.json",
+  JSON.stringify(fuseIndex.toJSON()),
+);

site_generator/src/git.ts

@@ -0,0 +1,6 @@
+import { simpleGit, SimpleGitOptions } from "simple-git";
+
+const options: Partial<SimpleGitOptions> = {
+  baseDir: `.`,
+};
+export const git = simpleGit(options);