Standardize formatting and fix session_start_time

.pre-commit-config.yaml

@@ -0,0 +1,56 @@
+default_language_version:
+  python: python3.9
+# files: "^(tests|src\/brain_lab)\/.*$"
+exclude: "^$"
+default_stages: [commit]
+repos:
+  # for debugging hook input arguments
+  - repo: meta
+    hooks:
+      - id: identity
+        exclude: (devcontainer\.json|\.code-workspace)$
+
+  # out-of-the-box hooks
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.2.0
+    hooks:
+      - id: check-case-conflict
+      - id: detect-private-key
+      - id: check-added-large-files
+        args: ["--maxkb=25000"]
+      - id: end-of-file-fixer
+        exclude: "^LICENSE|\\.csv$"
+      - id: mixed-line-ending
+        args: ["--fix=lf"] # replace 'auto' with 'lf' to enforce Linux/Mac line endings or 'crlf' for Windows
+        exclude: "^LICENSE|\\.csv$"
+
+  # run black code formatter on python files
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+        args:
+          - "--line-length"
+          - "88"
+
+  # run isort on python files to sort imports
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        args:
+          - "--profile"
+          - "black"
+
+  # run black on python code blocks in documentation files.
+  - repo: https://github.com/asottile/blacken-docs
+    rev: v1.12.1
+    hooks:
+      - id: blacken-docs
+        additional_dependencies: [black]
+
+  # strip out metadata for notebooks
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.5.0
+    hooks:
+      - id: nbstripout

Makefile

@@ -22,7 +22,3 @@ pypitest: clean sdist wheel
 
 clean:
 	rm -rf dist && rm -rf build && rm -rf *.egg-info
-
-
-
-

entrypoint.py

@@ -1,17 +1,19 @@
 import sys
-from ibl_pipeline.ingest import job
-from ibl_pipeline.process import populate_behavior, populate_wheel, populate_ephys
 
+from ibl_pipeline.ingest import job
+from ibl_pipeline.process import populate_behavior, populate_ephys, populate_wheel
 
-if __name__ == '__main__':
-    if sys.argv[1] == 'ingest':
+if __name__ == "__main__":
+    if sys.argv[1] == "ingest":
         job.populate_ingestion_tables(run_duration=-1)
-    elif sys.argv[1] == 'behavior':
+    elif sys.argv[1] == "behavior":
         populate_behavior.main(backtrack_days=3, run_duration=-1)
-    elif sys.argv[1] == 'wheel':
+    elif sys.argv[1] == "wheel":
         populate_wheel.main(backtrack_days=3, run_duration=-1)
-    elif sys.argv[1] == 'ephys':
+    elif sys.argv[1] == "ephys":
         populate_ephys.main(run_duration=-1)
     else:
-        raise ValueError(f'Usage error! Unknown argument {sys.argv[1]}. '
-                         f'Accepting: ingest|behavior|wheel|ephys')
+        raise ValueError(
+            f"Usage error! Unknown argument {sys.argv[1]}. "
+            f"Accepting: ingest|behavior|wheel|ephys"
+        )

howto.py

@@ -1,6 +1,7 @@
 # clean up external files
 import datajoint as dj
-schema = dj.schema('ibl_ephys')
+
+schema = dj.schema("ibl_ephys")
 
 # This only deletes entries that has been deleted before in the database.
-schema.external['ephys_local'].delete(delete_external_files=True)
+schema.external["ephys_local"].delete(delete_external_files=True)

ibl_pipeline/__init__.py

@@ -1,41 +1,23 @@
 import os
+import re
+from pathlib import Path
 
 import datajoint as dj
-
-_one = None
+from appdirs import user_cache_dir
 
 dj.config["enable_python_native_blobs"] = True
 
-mode = dj.config.get("custom", {}).get("database.mode", os.getenv("MODE", ""))
+mode = dj.config.get("custom", {}).get("database.mode", os.getenv("DJ_MODE", ""))
 
 if mode == "test":
     dj.config["database.prefix"] = "test_"
 elif mode == "update":
     dj.config["database.prefix"] = "update_"
 
 
-schema = dj.schema("ibl_storage")
-
-
-@schema
-class S3Access(dj.Manual):
-    definition = """
-    s3_id:  tinyint   # unique id for each S3 pair
-    ---
-    access_key: varchar(128)   # S3 access key
-    secret_key: varchar(128)   # S3 secret key
-    """
-
-
-# attempt to get S3 access/secret key from different sources
 access_key = os.getenv("S3_ACCESS")
 secret_key = os.getenv("S3_SECRET")
 
-if (access_key is None or secret_key is None) and len(S3Access.fetch()) > 0:
-    # if there are multiple entries in S3, it won't work
-    access_key, secret_key = S3Access.fetch1("access_key", "secret_key")
-
-
 if mode == "public":
     bucket = "ibl-dj-external-public"
     root = "/public"
@@ -63,17 +45,49 @@ class S3Access(dj.Manual):
 }
 
 
-try:
-    from one.api import OneAlyx
-except ImportError:
-    print("ONE-api not set up")
-    one = False
-else:
-    base_url = dj.config.get("custom", {}).get(
-        "database.alyx.url", os.getenv("ALYX_URL", None)
-    )
+def get_one_api_public(password=None, url="https://openalyx.internationalbrainlab.org"):
     try:
-        one = OneAlyx(base_url=base_url, silent=True)
-    except ConnectionError:
-        # by-pass error in removing the old format .one_params
-        one = OneAlyx(base_url=base_url, silent=True)
+        from one.api import OneAlyx
+    except ImportError:
+        print("'one-api' package not installed.")
+        one = None
+    else:
+        base_url = (
+            dj.config.get("custom", {}).get(
+                "database.alyx.url", os.getenv("ALYX_URL", None)
+            )
+            or url
+        )
+        cache_dir = (
+            Path(os.getenv("CACHE_DIR") or user_cache_dir("ibl"))
+            / "ONE"
+            / re.sub(r"^https*:/+", "", base_url)
+        )
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        try:
+            one = OneAlyx(
+                mode="remote",
+                wildcards=True,
+                base_url=base_url,
+                password=password or "international",
+                silent=True,
+                cache_dir=cache_dir,
+            )
+            one.refresh_cache("refresh")
+        except ConnectionError:
+            print(
+                "Could not connect to Alyx. Using 'openalyx.internationalbrainlab.org'"
+            )
+            one = OneAlyx(
+                mode="auto",
+                wildcards=True,
+                base_url="https://openalyx.internationalbrainlab.org",
+                password="international",
+                silent=True,
+                cache_dir=cache_dir,
+            )
+
+    return one
+
+
+one = get_one_api_public()

ibl_pipeline/acquisition.py

@@ -1,5 +1,5 @@
-from ibl_pipeline.acquisition_shared import *
 from ibl_pipeline import mode
+from ibl_pipeline.acquisition_shared import *
 
-if mode != 'public':
+if mode != "public":
     from ibl_pipeline.acquisition_internal import *

ibl_pipeline/acquisition_internal.py

@@ -1,24 +1,23 @@
 import datajoint as dj
-from ibl_pipeline import reference, subject, action
-from ibl_pipeline import mode
 
+from ibl_pipeline import action, mode, reference, subject
 
 # try to access parent schemas with virtual modules, if not created, import from package
 try:
-    action = dj.create_virtual_module('action', 'ibl_action')
+    action = dj.create_virtual_module("action", "ibl_action")
 except dj.DataJointError:
     from ibl_pipeline import action
 
 try:
-    acquisition = dj.create_virtual_module('acquisition', 'ibl_acquistion')
+    acquisition = dj.create_virtual_module("acquisition", "ibl_acquistion")
     Session = acquisition.Session
 except dj.DataJointError:
     from ibl_pipeline.acquisition import Session
 
-if mode == 'update':
-    schema = dj.schema('ibl_acquisition')
+if mode == "update":
+    schema = dj.schema("ibl_acquisition")
 else:
-    schema = dj.schema(dj.config.get('database.prefix', '') + 'ibl_acquisition')
+    schema = dj.schema(dj.config.get("database.prefix", "") + "ibl_acquisition")
 
 
 @schema

ibl_pipeline/acquisition_shared.py

@@ -1,20 +1,31 @@
-import datajoint as dj
 import datetime
-from tqdm import tqdm
+import re
 import uuid
 
-from ibl_pipeline import reference, subject, action
-from ibl_pipeline import mode, one
+import datajoint as dj
+from tqdm import tqdm
 
+from ibl_pipeline import action, mode, one, reference, subject
 
-alyxraw = dj.create_virtual_module('alyxraw', dj.config.get('database.prefix', '') + 'ibl_alyxraw')
+alyxraw = dj.create_virtual_module(
+    "alyxraw", dj.config.get("database.prefix", "") + "ibl_alyxraw"
+)
 
 
 # Map to the correct schema based on mode.
-if mode == 'update':
-    schema = dj.schema('ibl_acquisition')
+if mode == "update":
+    schema = dj.schema("ibl_acquisition")
 else:
-    schema = dj.schema(dj.config.get('database.prefix', '') + 'ibl_acquisition')
+    schema = dj.schema(dj.config.get("database.prefix", "") + "ibl_acquisition")
+
+
+_FLOAT_STR_REGEX = re.compile(r"\.[0-9]+$")
+
+
+def convert_time_str(tstr):
+    add_float = ".%f" if _FLOAT_STR_REGEX.search(tstr) else ""
+    dt_fmt = f"%Y-%m-%dT%H:%M:%S{add_float}"
+    return datetime.datetime.strptime(tstr, dt_fmt).strftime("%Y-%m-%d %H:%M:%S")
 
 
 @schema
@@ -42,36 +53,45 @@ def insert_with_alyx_rest(cls, backtrack_days=1, verbose=False):
         """
         date_cutoff = datetime.datetime.now() - datetime.timedelta(days=backtrack_days)
 
-        alyx_sessions = one.alyx.rest('sessions', 'list', django=f'start_time__gte,{date_cutoff}')
+        alyx_sessions = one.alyx.rest(
+            "sessions", "list", django=f"start_time__gte,{date_cutoff}"
+        )
 
         for alyx_session in tqdm(alyx_sessions):
-            if not subject.Subject & {'subject_nickname': alyx_session['subject']}:
+            if not subject.Subject & {"subject_nickname": alyx_session["subject"]}:
                 continue
 
             sess_key = {
-                'subject_uuid': (subject.Subject & {'subject_nickname': alyx_session['subject']}).fetch1('subject_uuid'),
-                'session_start_time': datetime.datetime.strptime(
-                    alyx_session["start_time"], '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y-%m-%d %H:%M:%S'),
-                }
+                "subject_uuid": (
+                    subject.Subject & {"subject_nickname": alyx_session["subject"]}
+                ).fetch1("subject_uuid"),
+                "session_start_time": convert_time_str(alyx_session["start_time"]),
+            }
 
-            sess_uuid = alyx_session['url'].split('/')[-1]
+            sess_uuid = alyx_session["url"].split("/")[-1]
 
-            if (cls & sess_key) or (alyxraw.AlyxRaw & {'uuid': sess_uuid}):
+            if (cls & sess_key) or (alyxraw.AlyxRaw & {"uuid": sess_uuid}):
                 # If this session is already in AlyxRaw, skip, as it will get inserted into Session in this ingestion cycle
                 continue
 
             if verbose:
-                print(f'\tInserting new session directly from Alyx: {alyx_session["subject"]}, {sess_key["session_start_time"]}')
-
-            cls.insert1({**sess_key,
-                         'session_uuid': uuid.UUID(sess_uuid),
-                         'session_number': alyx_session['number'],
-                         'session_end_time': None,
-                         'session_lab': alyx_session['lab'],
-                         'session_location': None,
-                         'task_protocol': alyx_session['task_protocol'],
-                         'session_type': None,
-                         'session_narrative': None})
+                print(
+                    f'\tInserting new session directly from Alyx: {alyx_session["subject"]}, {sess_key["session_start_time"]}'
+                )
+
+            cls.insert1(
+                {
+                    **sess_key,
+                    "session_uuid": uuid.UUID(sess_uuid),
+                    "session_number": alyx_session["number"],
+                    "session_end_time": None,
+                    "session_lab": alyx_session["lab"],
+                    "session_location": None,
+                    "task_protocol": alyx_session["task_protocol"],
+                    "session_type": None,
+                    "session_narrative": None,
+                }
+            )
 
 
 @schema

ibl_pipeline/action.py

@@ -1,6 +1,5 @@
-from ibl_pipeline.action_shared import *
 from ibl_pipeline import mode
+from ibl_pipeline.action_shared import *
 
-
-if mode != 'public':
+if mode != "public":
     from ibl_pipeline.action_internal import *

ibl_pipeline/action_internal.py

@@ -1,14 +1,14 @@
-import datajoint as dj
-from ibl_pipeline import reference, subject
-from ibl_pipeline.action_shared import ProcedureType
 import os
 
-mode = dj.config.get('custom', {}).get('database.mode', "")
+import datajoint as dj
+
+from ibl_pipeline import mode, reference, subject
+from ibl_pipeline.action_shared import ProcedureType
 
-if mode == 'update':
-    schema = dj.schema('ibl_action')
+if mode == "update":
+    schema = dj.schema("ibl_action")
 else:
-    schema = dj.schema(dj.config.get('database.prefix', '') + 'ibl_action')
+    schema = dj.schema(dj.config.get("database.prefix", "") + "ibl_action")
 
 
 @schema