Merge pull request #83 from LSSTDESC/u/stuart/add_schema_to_rootdir

Add schema name to the shared space path

.github/workflows/ci.yml

@@ -137,14 +137,11 @@ jobs:
         run: |
           cd tests/end_to_end_tests
 
-          # Register database entries using dataregistry package
-          python create_test_entries.py
-
           # Register more database entries using the CLI
           bash create_test_entries_cli.sh
 
           # Run some test queries
-          pytest -v test_query.py
+          pytest -v test_*.py
 
   sqlite-end-to-end-tests:
     runs-on: ubuntu-latest
@@ -196,11 +193,8 @@ jobs:
         run: |
           cd tests/end_to_end_tests
 
-          # Register database entries using dataregistry package
-          python create_test_entries.py
-
           # Register more database entries using the CLI
           bash create_test_entries_cli.sh
 
           # Run some test queries
-          pytest -v test_query.py
+          pytest -v -m "not skip" test_*.py

src/cli/register.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 import os
 from dataregistry import DataRegistry
 
@@ -23,6 +24,10 @@ def register_dataset(args):
     found in `src/cli/cli.py` or by running `dregs --help`.
     """
 
+    # Convert to a datetime object (needed for SQLite)
+    if args.creation_date is not None:
+        args.creation_date = datetime.strptime(args.creation_date, "%Y-%m-%d")
+
     # Connect to database.
     datareg = DataRegistry(
         config_file=args.config_file,

src/dataregistry/_version.py

@@ -1 +1 @@
-__version__ = "0.2.2"
+__version__ = "0.3.0"

src/dataregistry/query.py

@@ -429,6 +429,7 @@ def get_dataset_absolute_path(self, dataset_id, schema=None):
                 results["dataset.owner_type"][0],
                 results["dataset.owner"][0],
                 results["dataset.relative_path"][0],
+                schema=schema,
                 root_dir=self._root_dir,
             )
         else:

src/dataregistry/registrar.py

@@ -170,7 +170,11 @@ def _handle_data(self, relative_path, old_location, owner, owner_type, verbose):
 
         # Get destination directory in data registry.
         dest = _form_dataset_path(
-            owner_type, owner, relative_path, root_dir=self._root_dir
+            owner_type,
+            owner,
+            relative_path,
+            schema=self._schema,
+            root_dir=self._root_dir,
         )
 
         # Is the data already on location, or coming from somewhere new?

src/dataregistry/registrar_util.py

@@ -55,14 +55,15 @@ def _parse_version_string(version, with_suffix=False):
     return d
 
 
-def _form_dataset_path(owner_type, owner, relative_path, root_dir=None):
+def _form_dataset_path(owner_type, owner, relative_path, schema=None, root_dir=None):
     """
     Construct full (or relative) path to dataset in the data registry.
 
-    Path will have the format if `root_dir` is None:
+    When schema and root_dir are not None, the full path is returned:
+        <root_dir>/<schema>/<owner_type>/<owner>/<relative_path>
+
+    When schema and root_dir are ommited, the relative path is returned:
         <owner_type>/<owner>/<relative_path>
-    or if `root_dir` is not None:
-        <root_dir>/<owner_type>/<owner>/<relative_path>
 
     Parameters
     ----------
@@ -72,17 +73,24 @@ def _form_dataset_path(owner_type, owner, relative_path, root_dir=None):
         Owner of dataset
     relative_path : str
         Relative path within the data registry
-    root_dir : str
+    schema : str, optional
+        Schema we are connected to
+    root_dir : str, optional
         Root directory of data registry
+    dialect : str, optional
+        SQL dialect, e.g postgres or sqlite
 
     Returns
     -------
     to_return : str
-        Full path of dataset in the data registry
+        Full (or relative) path of dataset in the data registry
     """
+
     if owner_type == "production":
         owner = "production"
     to_return = os.path.join(owner_type, owner, relative_path)
+    if schema:
+        to_return = os.path.join(schema, to_return)
     if root_dir:
         to_return = os.path.join(root_dir, to_return)
     return to_return