Merge pull request #93 from dandi/index_dandisets_with_notebooks

Index dandisets with notebooks
dandi · Jul 1, 2024 · eaa0e34 · eaa0e34
2 parents 049a8d4 + ee5b8ed
commit eaa0e34
Show file tree

Hide file tree

Showing 4 changed files with 234 additions and 1 deletion.
diff --git a/.github/scripts/metadata-collector-script.py b/.github/scripts/metadata-collector-script.py
@@ -0,0 +1,108 @@
+import os
+from typing import List, Dict, Any, Optional
+from jinja2 import Environment, FileSystemLoader
+from dandi.dandiapi import DandiAPIClient
+
+
+def get_dandiset_metadata(dandiset_id: str) -> Optional[Dict[str, Any]]:
+    """
+    Fetch metadata for a given dandiset ID.
+
+    Parameters
+    ----------
+    dandiset_id : str
+        The ID of the dandiset to fetch metadata for.
+
+    Returns
+    -------
+    Optional[Dict[str, Any]]
+        A dictionary containing the dandiset metadata if successful, None otherwise.
+    """
+    with DandiAPIClient() as client:
+        try:
+            dandiset = client.get_dandiset(dandiset_id)
+            metadata = dandiset.get_raw_metadata()
+            return metadata
+        except Exception as e:
+            print(f"Error fetching metadata for dandiset {dandiset_id}: {str(e)}")
+            return None
+
+
+def find_notebooks(folder: str) -> List[str]:
+    """
+    Recursively find all Jupyter notebooks in a given folder.
+
+    Parameters
+    ----------
+    folder : str
+        The path to the folder to search in.
+
+    Returns
+    -------
+    List[str]
+        A list of relative paths to the found notebooks.
+    """
+    notebooks = []
+    for root, _, files in os.walk(folder):
+        for file in files:
+            if file.endswith('.ipynb'):
+                rel_path = os.path.relpath(os.path.join(root, file), folder)
+                notebooks.append(rel_path)
+    return notebooks
+
+
+def collect_metadata() -> List[Dict[str, Any]]:
+    """
+    Collect metadata and notebook information for all dandisets in the current directory.
+
+    Returns
+    -------
+    List[Dict[str, Any]]
+        A list of dictionaries, each containing information about a dandiset,
+        sorted by dandiset ID.
+    """
+    dandisets = []
+    for folder in os.listdir('.'):
+        if os.path.isdir(folder) and folder.isdigit():
+            metadata = get_dandiset_metadata(folder)
+            if metadata:
+                notebooks = find_notebooks(folder)
+                dandisets.append({
+                    'id': folder,
+                    'metadata': metadata,
+                    'notebooks': notebooks
+                })
+
+    dandisets.sort(key=lambda x: x['id'])
+    return dandisets
+
+
+def render_webpage(dandisets: List[Dict[str, Any]]) -> None:
+    """
+    Render the webpage using the collected dandiset information.
+
+    Parameters
+    ----------
+    dandisets : List[Dict[str, Any]]
+        A list of dictionaries containing information about each dandiset.
+
+    Returns
+    -------
+    None
+    """
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+    template_dir = os.path.join(current_dir, '..', 'templates')
+
+    env = Environment(loader=FileSystemLoader(template_dir))
+    template = env.get_template('index.html')
+
+    output = template.render(dandisets=dandisets)
+
+    output_dir = os.path.join(current_dir, '..', '..', 'output')
+    os.makedirs(output_dir, exist_ok=True)
+    with open(os.path.join(output_dir, 'index.html'), 'w') as f:
+        f.write(output)
+
+if __name__ == "__main__":
+    dandisets = collect_metadata()
+    render_webpage(dandisets)
diff --git a/.github/templates/index.html b/.github/templates/index.html
@@ -0,0 +1,93 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>DANDI Datasets</title>
+    <style>
+        body {
+            font-family: 'Arial', sans-serif;
+            line-height: 1.6;
+            margin: 0;
+            padding: 20px;
+        }
+        .container {
+            max-width: 1200px;
+            margin: 0 auto;
+        }
+        h1 {
+            text-align: center;
+            margin-bottom: 30px;
+        }
+        .dandiset {
+            border-radius: 8px;
+            box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+            margin-bottom: 30px;
+            padding: 20px;
+            transition: box-shadow 0.3s ease;
+        }
+        .dandiset:hover {
+            box-shadow: 0 4px 8px rgba(0, 0, 0, 0.15);
+        }
+        .dandiset h2 {
+            margin-top: 0;
+            margin-bottom: 15px;
+        }
+        .dandiset p {
+            margin-bottom: 10px;
+        }
+        .dandiset a {
+            text-decoration: none;
+            font-weight: bold;
+        }
+        .dandiset a:hover {
+            text-decoration: underline;
+        }
+        .notebooks {
+            margin-top: 15px;
+        }
+        .notebooks h3 {
+            font-size: 1.1em;
+            margin-bottom: 10px;
+        }
+        .notebooks ul {
+            list-style-type: disc;
+            padding-left: 20px;
+        }
+        .notebooks li {
+            margin-bottom: 5px;
+        }
+        @media (max-width: 768px) {
+            body {
+                padding: 10px;
+            }
+            .dandiset {
+                padding: 15px;
+            }
+        }
+    </style>
+</head>
+<body>
+<div class="container">
+    <h1>DANDI Datasets</h1>
+    {% for dandiset in dandisets %}
+    <div class="dandiset">
+        <h2>{{ dandiset.metadata.name }}</h2>
+        <p><strong>ID:</strong> {{ dandiset.id }}</p>
+        <p><strong>Description:</strong> {{ dandiset.metadata.description }}</p>
+        <p><a href="https://dandiarchive.org/dandiset/{{ dandiset.id }}" target="_blank">View on DANDI Archive</a></p>
+        {% if dandiset.notebooks %}
+        <div class="notebooks">
+            <h3>Notebooks:</h3>
+            <ul>
+                {% for notebook in dandiset.notebooks %}
+                <li><a href="https://github.com/dandi/example-notebooks/blob/master/{{ dandiset.id }}/{{ notebook }}" target="_blank">{{ notebook }}</a></li>
+                {% endfor %}
+            </ul>
+        </div>
+        {% endif %}
+    </div>
+    {% endfor %}
+</div>
+</body>
+</html>
diff --git a/.github/workflows/index_workflow.yml b/.github/workflows/index_workflow.yml
@@ -0,0 +1,30 @@
+name: DANDI Metadata Collector
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  collect-and-render:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install requests jinja2 dandi
+    
+    - name: Run metadata collector and renderer
+      run: python .github/scripts/collect_and_render.py
+
+    - name: Deploy to GitHub Pages
+      uses: peaceiris/actions-gh-pages@v3
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ./output
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
 .ipynb_checkpoints
 *.DS_Store
-*__pycache__/
+*__pycache__/
+*.ipynb_checkpoints/
+output/