Added action, file rename

job name error

revert to default...

Take from example

Create test.yml

run in yest.yml

indenting

pip install in action

Keep going if you encounter errors specific to a repo

git commit the changes

update 100 random entries every hour

.github/workflows/update-metadata.yml

@@ -9,13 +9,22 @@ on:
     branches: 
   pull_request:
     branches: 
+
+  #schedule:
+  #  - cron: "13 * * * *"    
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:  
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
   parse-metadata:
+    # The type of runner that the job will run on  
     runs-on: ubuntu-latest
 
+    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it    
       - uses: actions/checkout@v3
 
       - uses: actions/setup-python@v4
@@ -35,6 +44,20 @@ jobs:
           git config --local user.email "github-actions[bot]@users.noreply.github.com"
           git config --local user.name "github-actions[bot]"
           git commit -m "Update statistics" -a
+          
+	  # Runs a set of commands using the runners shell
+      - name: Update files
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          cd code
+          pip install -r requirements.txt
+          cd ..
+          python3 code/github_import.py
+
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git commit -m "Update metadata" -a          
 
       - name: Push changes
         uses: ad-m/github-push-action@master

code/github_import.py

@@ -9,6 +9,7 @@
 
 import os
 import json
+from random import sample
 from utils import constants as c, utils, osg, osg_parse, osg_github
 
 gh_entries_file = os.path.join(c.code_path, 'github_entries.txt')
@@ -66,17 +67,20 @@ def github_import():
     """
     Import various information from GitHub repositories (like contributors) or stars for GitHub repos
     """
-    private_properties = json.loads(utils.read_text(c.private_properties_file))
 
     files = json.loads(utils.read_text(gh_entries_file))
 
+    # Github rate limiting limits us to 1000 queries an hour, currently let's limit it to 100.
+    if len(files) > 100:
+        files = sample(files, 100)
+
+
     all_developers = osg.read_developers()
     print(' {} developers read'.format(len(all_developers)))
 
-    # all exceptions that happen will be eaten (but will end the execution)
-    try:
-        # loop over each entry
-        for index, file in enumerate(files):
+    # loop over each entry
+    for index, file in enumerate(files):
+        try:
             print(' process {} ({})'.format(file, index))
 
             # read entry
@@ -89,8 +93,12 @@ def github_import():
             repos = [x for x in repos if x not in ignored_repos]
             for repo in repos:
                 print('  GH repo {}'.format(repo))
+                token = os.environ["GITHUB_TOKEN"]
+                if not token:
+                    private_properties = json.loads(utils.read_text(c.private_properties_file))
+                    token = private_properties['github-token']
 
-                info = osg_github.retrieve_repo_info(repo, token=private_properties['github-token'])
+                info = osg_github.retrieve_repo_info(repo, token=token)
                 if info is None:
                     continue
 
@@ -182,14 +190,15 @@ def github_import():
 
             entry['Code repository'] = code_repositories
             osg.write_entry(entry)
-    except:
-        raise
-    finally:
-        # shorten file list
-        utils.write_text(gh_entries_file, json.dumps(files[index:], indent=1))
+        except:
+            print(f"Error processing repo {file}")
+            pass # Keep going to other entries
+
+    # shorten file list
+    utils.write_text(gh_entries_file, json.dumps(files[index:], indent=1))
 
-        osg.write_developers(all_developers)
-        print('developers database updated')
+    osg.write_developers(all_developers)
+    print('developers database updated')
 
 
 def github_starring_synchronization():
@@ -234,10 +243,10 @@ def github_starring_synchronization():
 
 if __name__ == "__main__":
     # collect entries (run this only once)
-    # collect_github_entries()
+    collect_github_entries()
 
     # import information from gh
     github_import()
 
-    # which GitHub repos have I not yet starred
+    # which github repos haven't I starred
     # github_starring_synchronization()