Soomin/puzzles missing games

app/templating/AssetHelper.scala

@@ -108,11 +108,50 @@ trait AssetHelper { self: I18nHelper with SecurityHelper =>
     )
   }
 
+  def basicCspNoSelf(implicit req: RequestHeader): ContentSecurityPolicy = {
+    val assets = assetDomain.value
+    val sockets = socketDomains map { socketDomain =>
+      val protocol = if (req.secure) "wss://" else "ws://"
+      s"$protocol$socketDomain"
+    }
+    val localDev = !req.secure ?? List("http://127.0.0.1:3000")
+    /*
+    ContentSecurityPolicy(
+      defaultSrc = List("'self'", assets),
+      connectSrc =
+        "'self'" :: assets :: "*.logrocket.io" :: "*.lr-ingest.io" :: "*.lr-in-prod.com" :: sockets ::: env.explorerEndpoint :: env.tablebaseEndpoint :: localDev,
+      styleSrc = List("'self'", "'unsafe-inline'", assets),
+      frameSrc = List("'self'", assets, "www.youtube.com", "player.twitch.tv"),
+      workerSrc = List("'self'", "*", "blob:", assets),
+      imgSrc = List("data:", "*"),
+      scriptSrc = List("'self'", assets, "cdn.lr-in-prod.com", "cdn.logrocket.io", "cdn.lr-ingest.io", "; child-src 'self' blob:"),
+      baseUri = List("'none'"),
+    )
+     */
+
+    ContentSecurityPolicy(
+      defaultSrc = List("'self'", assets),
+      connectSrc =
+        "'self'" :: "https://lichess.org" :: "*.lichess.org" :: assets :: "*.logrocket.io" :: "*.logrocket.com" :: "*.lr-ingest.io" :: "*.lr-in-prod.com" :: sockets ::: env.explorerEndpoint :: env.tablebaseEndpoint :: localDev,
+      styleSrc = List("'self'", "'unsafe-inline'", assets),
+      frameSrc = List("'self'", assets, "www.youtube.com", "player.twitch.tv"),
+      workerSrc = List("'self'", "blob:", "data:", assets),
+      imgSrc = List("data:", "*"),
+      scriptSrc = List("'self'", "cdn.lr-in-prod.com", "cdn.logrocket.io", "cdn.lr-ingest.io", assets),
+      baseUri = List("'none'")
+    )
+  }
+
   def defaultCsp(implicit ctx: Context): ContentSecurityPolicy = {
     val csp = basicCsp(ctx.req)
     ctx.nonce.fold(csp)(csp.withNonce(_))
   }
 
+  def defaultCspNoSelf(implicit ctx: Context): ContentSecurityPolicy = {
+    val csp = basicCspNoSelf(ctx.req)
+    ctx.nonce.fold(csp)(csp.withNonce(_))
+  }
+
   def embedJsUnsafe(js: String)(implicit ctx: Context): Frag =
     raw {
       val nonce = ctx.nonce ?? { nonce =>

app/views/puzzle/direct.scala

@@ -19,7 +19,7 @@ object direct {
     views.html.base.layout(
       title = if (isStreak) "Puzzle Streak" else trans.puzzles.txt(),
       moreCss = frag(
-        cssTag("puzzle.direct"),
+        cssTag("puzzle.direct")
       ),
       moreJs = frag(
         jsModule("puzzle.direct"),
@@ -35,13 +35,13 @@ object direct {
               .add("difficulty" -> difficulty.map(_.key))
           )})""")
       ),
-      csp = defaultCsp.withWebAssembly.withAnyWs.some,
+      csp = defaultCspNoSelf.some,
       chessground = false,
       zoomable = false,
       playing = true
     ) {
       main(cls := "puzzle")(
-        div(cls := "puzzle__board main-board")(chessgroundBoard),
+        div(cls := "puzzle__board main-board")(chessgroundBoard)
       )
     }
   }

app/views/puzzle/show.scala

@@ -39,7 +39,7 @@ object show {
               .add("difficulty" -> difficulty.map(_.key))
           )})""")
       ),
-      csp = defaultCsp.withWebAssembly.withAnyWs.some,
+      csp = defaultCspNoSelf.some,
       chessground = false,
       openGraph = lila.app.ui
         .OpenGraph(

bin/mongodb/puzzle-regen-paths.js

@@ -0,0 +1,236 @@
+/* Generates and saves a new generation of puzzle paths.
+ * Drops the previous generation.
+ *
+ * mongo <IP>:<PORT>/<DB> mongodb-puzzle-regen-paths.js
+ *
+ * Must run on the puzzle database.
+ * Should run every 60 minutes.
+ * Should complete within 3 minutes.
+ * OK to run many times in a row.
+ * OK to skip runs.
+ * NOT OK to run concurrently.
+ *
+ * might require this mongodb config: (https://jira.mongodb.org/browse/SERVER-44174)
+ * setParameter:
+ *   internalQueryMaxPushBytes: 314572800
+ */
+
+const puzzleColl = db.puzzle2_puzzle;
+const pathColl = db.puzzle2_path_new;
+const verbose = false;
+const maxRatingBuckets = 12;
+const mixRatingBuckets = 20;
+const maxPathLength = 500;
+const maxPuzzlesPerTheme = 2 * 1000 * 1000; // reduce to 500000 to avoid memory restrictions in some envs (!?)
+
+const generation = Date.now();
+
+const tiers = [
+  ['top', 20 / 100],
+  ['good', 50 / 100],
+  ['all', 95 / 100],
+];
+
+const themes = db.puzzle2_puzzle.distinct('themes', {});
+
+function chunkify(a, n) {
+  let len = a.length,
+    out = [],
+    i = 0,
+    size;
+  if (len % n === 0) {
+    size = Math.floor(len / n);
+    while (i < len) {
+      out.push(a.slice(i, (i += size)));
+    }
+  } else
+    while (i < len) {
+      size = Math.ceil((len - i) / n--);
+      out.push(a.slice(i, (i += size)));
+    }
+  return out;
+}
+const padRating = r => (r < 1000 ? '0' : '') + r;
+
+themes.concat(['mix']).forEach(theme => {
+  const selector = {
+    themes:
+      theme == 'mix'
+        ? {
+            $ne: 'equality',
+          }
+        : theme == 'equality'
+        ? 'equality'
+        : {
+            $eq: theme,
+            $ne: 'equality',
+          },
+  };
+
+  const nbPuzzles = puzzleColl.count(selector);
+
+  if (!nbPuzzles) return [];
+
+  const themeMaxPathLength = Math.max(10, Math.min(maxPathLength, Math.round(nbPuzzles / 200)));
+  const nbRatingBuckets =
+    theme == 'mix'
+      ? mixRatingBuckets
+      : Math.max(3, Math.min(maxRatingBuckets, Math.round(nbPuzzles / themeMaxPathLength / 20)));
+
+  if (verbose)
+    print(
+      `theme: ${theme}, puzzles: ${nbPuzzles}, path length: ${themeMaxPathLength}, rating buckets: ${nbRatingBuckets}`
+    );
+
+  let bucketIndex = 0;
+
+  db.puzzle2_puzzle
+    .aggregate(
+      [
+        {
+          $match: selector,
+        },
+        {
+          $limit: maxPuzzlesPerTheme,
+        },
+        {
+          $bucketAuto: {
+            buckets: nbRatingBuckets,
+            groupBy: '$glicko.r',
+            output: {
+              puzzle: {
+                $push: {
+                  id: '$_id',
+                  vote: '$vote',
+                },
+              },
+            },
+          },
+        },
+        {
+          $unwind: '$puzzle',
+        },
+        {
+          $sort: {
+            'puzzle.vote': -1,
+          },
+        },
+        {
+          $group: {
+            _id: '$_id',
+            total: {
+              $sum: 1,
+            },
+            puzzles: {
+              $push: '$puzzle.id',
+            },
+          },
+        },
+        {
+          $facet: tiers.reduce(
+            (facets, [name, ratio]) => ({
+              ...facets,
+              ...{
+                [name]: [
+                  {
+                    $project: {
+                      total: 1,
+                      puzzles: {
+                        $slice: [
+                          '$puzzles',
+                          {
+                            $round: {
+                              $multiply: ['$total', ratio],
+                            },
+                          },
+                        ],
+                      },
+                    },
+                  },
+                  {
+                    $unwind: '$puzzles',
+                  },
+                  {
+                    $sample: {
+                      // shuffle
+                      size: 9999999,
+                    },
+                  },
+                  {
+                    $group: {
+                      _id: '$_id',
+                      puzzles: {
+                        $addToSet: '$puzzles',
+                      },
+                    },
+                  },
+                  {
+                    $sort: {
+                      '_id.min': 1,
+                    },
+                  },
+                  {
+                    $addFields: {
+                      tier: name,
+                    },
+                  },
+                ],
+              },
+            }),
+            {}
+          ),
+        },
+        {
+          $project: {
+            bucket: {
+              $concatArrays: tiers.map(t => '$' + t[0]),
+            },
+          },
+        },
+        {
+          $unwind: '$bucket',
+        },
+        {
+          $replaceRoot: {
+            newRoot: '$bucket',
+          },
+        },
+      ],
+      {
+        allowDiskUse: true,
+        comment: 'regen-paths',
+      }
+    )
+    .forEach(bucket => {
+      const isFirstOfTier = bucketIndex % nbRatingBuckets == 0;
+      const isLastOfTier = bucketIndex % nbRatingBuckets == nbRatingBuckets - 1;
+      const pathLength = Math.max(10, Math.min(maxPathLength, Math.round(bucket.puzzles.length / 30)));
+      const ratingMin = isFirstOfTier ? 100 : Math.ceil(bucket._id.min);
+      const ratingMax = isLastOfTier ? 9999 : Math.floor(bucket._id.max);
+      const nbPaths = Math.max(1, Math.floor(bucket.puzzles.length / pathLength));
+      const paths = chunkify(bucket.puzzles, nbPaths);
+      // print(`  ${theme} ${bucket.tier} ${ratingMin}->${ratingMax} puzzles: ${bucket.puzzles.length} pathLength: ${pathLength} paths: ${paths.length}`);
+
+      pathColl.insert(
+        paths.map((ids, j) => ({
+          _id: `${theme}_${bucket.tier}_${padRating(ratingMin)}-${padRating(ratingMax)}_${generation}_${j}`,
+          min: `${theme}_${bucket.tier}_${padRating(ratingMin)}`,
+          max: `${theme}_${bucket.tier}_${padRating(ratingMax)}`,
+          ids,
+          tier: bucket.tier,
+          theme: theme,
+          gen: generation,
+        })),
+        {
+          ordered: false,
+        }
+      );
+      bucketIndex++;
+    });
+});
+
+pathColl.remove({
+  gen: {
+    $ne: generation,
+  },
+});

modules/common/src/main/HTTPRequest.scala

@@ -28,7 +28,9 @@ object HTTPRequest {
     "capacitor://localhost", // ios
     "ionic://localhost",     // ios
     "http://localhost",      // android
-    "http://localhost:8080"  // local dev
+    "http://localhost:8080",  // local dev
+    "http://localhost:9663",
+    "http://localhost:3000"
   )
 
   def appOrigin(req: RequestHeader) = origin(req) filter appOrigins

modules/puzzle/src/main/Env.scala

@@ -4,6 +4,7 @@ import com.softwaremill.macwire._
 import io.methvin.play.autoconfig._
 import play.api.Configuration
 import scala.concurrent.duration._
+import play.api.libs.ws.StandaloneWSClient
 
 import lila.common.config._
 import lila.db.AsyncColl
@@ -19,6 +20,7 @@ private class PuzzleConfig(
 @Module
 final class Env(
     appConfig: Configuration,
+    ws: StandaloneWSClient,
     renderer: lila.hub.actors.Renderer,
     historyApi: lila.history.HistoryApi,
     lightUserApi: lila.user.LightUserApi,
@@ -46,6 +48,8 @@ final class Env(
 
   lazy val jsonView = wire[JsonView]
 
+  lazy val retrieveGameApi: RetrieveGameApi = wire[RetrieveGameApi]
+
   private lazy val pathApi = wire[PuzzlePathApi]
 
   private lazy val trustApi = wire[PuzzleTrustApi]