feat: support skipped or unlimited testbed

also add `-1` support for query limit

.github/workflows/ci.yml

@@ -53,22 +53,22 @@ jobs:
   testbed:
     needs: index
     # Run even if `index` is skipped
-    if: (!failure() && !cancelled())
+    if: (!failure() && !cancelled() && inputs.testbed-size != 0)
     name: Testbed
     uses: ./.github/workflows/testbed.yaml
     secrets: inherit
     with:
       index-repo: ${{ inputs.index-repo }}
-      testbed-size: ${{ inputs.testbed-size || 0 }}
+      testbed-size: ${{ inputs.testbed-size || (github.ref_name == 'master' && 200 || 0) }}
       toolchain: ${{ inputs.toolchain }}
       update-index: ${{ inputs.save-testbed == true }}
   pages:
     needs: [index, testbed]
-    # Run even if `index` is skipped
+    # Run even if `index` or `testbed` is skipped
     if: (!failure() && !cancelled())
     name: Pages
     uses: ./.github/workflows/pages.yaml
     secrets: inherit
     with:
       index-repo: ${{ inputs.index-repo }}
-      testbed-artifact: ${{ !(inputs.index-repo && inputs.save-testbed) }}
+      testbed-artifact: ${{ !(inputs.index-repo && inputs.save-testbed) && inputs.testbed-size != 0 }}

.github/workflows/index.yaml

@@ -43,7 +43,7 @@ jobs:
         run: |
           ./query.py -v \
             ${{ inputs.repository && '-D index' || '-o index.json' }} \
-            -L ${{ (!inputs.repository && github.ref_name != 'master') && 100 || 0 }}
+            -L ${{ (!inputs.repository && github.ref_name != 'master') && 100 || -1 }}
         env:
           GH_TOKEN: ${{ github.token }}
       - name: Update Index Repository

.github/workflows/testbed.yaml

@@ -87,8 +87,8 @@ jobs:
           ./testbed-create.py -o matrix.json \
             ${{ inputs.index-repo && 'index' || 'index.json' }} \
             ${{ inputs.toolchain || steps.find-toolchain.outputs.toolchain }}  \
-            -n ${{inputs.testbed-size || (github.ref_name == 'master' && 200 || 15)}} \
-            -X ${{github.ref_name == 'master' && 'testbed-exclusions.txt' || 'testbed-dev-exclusions.txt'}}
+            -n ${{ inputs.testbed-size || 10 }} \
+            -X ${{ github.ref_name == 'master' && 'testbed-exclusions.txt' || 'testbed-dev-exclusions.txt' }}
       - name: Upload Matrix
         uses: actions/upload-artifact@v3
         with:

query.py

@@ -46,7 +46,7 @@ def query_repo_data(repoIds: 'list[str]') -> dict:
 # Thus, the strategy used here will need to change when we hit that limit.
 def query_lake_repos(limit: int) -> 'list[str]':
   query='filename:lakefile.lean path:/'
-  if limit == 0:
+  if limit <= 0:
     out = capture_cmd(
       'gh', 'api', 'search/code',
       '--paginate', '--cache', '1h',

testbed-create.py

@@ -36,7 +36,9 @@ def create_entry(pkg: 'dict[str, any]'):
   pkgs = load_index(args.index)
   pkgs = filter(lambda pkg: pkg['fullName'] not in exclusions, pkgs)
   pkgs = map(create_entry, pkgs)
-  pkgs = list(itertools.islice(pkgs, args.num))
+  if args.num != -1:
+    pkgs = itertools.islice(pkgs, args.num)
+  pkgs = list(pkgs)
 
   if args.output is None:
     print(json.dumps(pkgs))