new version

nsdf-fabric · Oct 21, 2022 · e0156f6 · e0156f6
1 parent 4067672
commit e0156f6
Show file tree

Hide file tree

Showing 10 changed files with 225 additions and 579 deletions.
diff --git a/jupyter/nsdf-material-science-workflow.ipynb b/jupyter/nsdf-material-science-workflow.ipynb
@@ -52,21 +52,6 @@
     "\t\tendpoint_url=os.environ[\"AWS_ENDPOINT_URL\"])\n",
     "\n",
     "# ////////////////////////////////////////////////////////\n",
-    "def ListObjects(bucket, folder):\n",
-    "\t\"\"\"\n",
-    "\tList of Object Storage given a certain prefix, useful for ls-like finding of objects\n",
-    "\t\"\"\"\n",
-    "\tt1=time.time()\n",
-    "\tpaginator = s3.get_paginator('list_objects_v2')\n",
-    "\tpages = paginator.paginate(Bucket=bucket, Prefix=folder) if folder is not None else paginator.paginate(Bucket=bucket)\n",
-    "\tret=[]\n",
-    "\tfor page in pages:\n",
-    "\t\tfor obj in page['Contents'] if 'Contents' in page else []:\n",
-    "\t\t\tret.append(obj['Key'])\n",
-    "\tprint(f\"ListObjects {folder} in {time.time()-t1} seconds\")\n",
-    "\treturn ret\n",
-    "\n",
-    "# ////////////////////////////////////////////////////////\n",
     "def DownloadObject(bucket, key, local, force=False):\n",
     "\t\"\"\"\n",
     "\tDownload object from S3\n",
@@ -517,7 +502,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.8.10 64-bit",
+   "display_name": "Python 3.9.11 64-bit",
    "language": "python",
    "name": "python3"
   },
@@ -531,12 +516,12 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.10"
+   "version": "3.9.11"
   },
   "orig_nbformat": 4,
   "vscode": {
    "interpreter": {
-    "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
+    "hash": "81794d4967e6c3204c66dcd87b604927b115b27c00565d3d43f05ba2f3a2cb0d"
    }
   }
  },

diff --git a/nsdf/convert/openvisus.py b/nsdf/convert/openvisus.py
@@ -4,7 +4,7 @@
 import OpenVisus as ov
 
 from nsdf.kernel import logger, FileExists, TouchFile, SafeReplaceFile,rmdir
-from nsdf.s3 import S3, S3Sync
+from nsdf.s3 import S3
 
 # ////////////////////////////////////////////////////////////////////////////////////////
 """
@@ -409,7 +409,7 @@ def ConvertImageStack(
 
 	# sync with remote source
 	if src["remote"]:
-		S3Sync(logger, f"cloud-to-local",src["remote"], src["local"])
+		S3.sync(f"cloud-to-local",src["remote"], src["local"])
 
 	# convert
 	if True:
@@ -444,7 +444,7 @@ def ConvertImageStack(
 
 	# sync with remote destination
 	if dst["remote"]:
-		S3Sync(logger, f"local-to-cloud", os.path.dirname(dst['local']),os.path.dirname(dst['remote'])) 
+		S3.sync(f"local-to-cloud", os.path.dirname(dst['local']),os.path.dirname(dst['remote'])) 
 
 	# remove IDX (dst[local] is the idxfilename)
 	if not dst["keep-local"]:
@@ -480,7 +480,7 @@ def CopyDataset(
 
 	# sync with remote source
 	if src["remote"]:
-		S3Sync(logger, f"cloud-to-local",src["remote"], src["local"])
+		S3.sync(f"cloud-to-local",src["remote"], src["local"])
 
 	SRC=ov.LoadIdxDataset(src["local"])
 
@@ -561,7 +561,7 @@ def CopyDataset(
 
 	# sync with remote destination
 	if dst["remote"]:
-		S3Sync(logger, f"local-to-cloud", os.path.dirname(dst['local']),os.path.dirname(dst['remote'])) 
+		S3.sync(f"local-to-cloud", os.path.dirname(dst['local']),os.path.dirname(dst['remote'])) 
 
 	# remove local destination (dst[local] is the idx filename)
 	if not dst["keep-local"]: 

diff --git a/nsdf/material_science_workflow/__main__.py b/nsdf/material_science_workflow/__main__.py
@@ -15,7 +15,7 @@
 from prefect import Flow, Parameter, Task, task, unmapped
 
 from nsdf.kernel import logger, LoadYaml, rmfile, GetPackageFilename, RunCommand, SetupLogger, LoadYaml
-from nsdf.s3 import S3,S3Sync, S3ParseUrl
+from nsdf.s3 import S3
 from nsdf.distributed import NSDFDaskCluster
 
 
@@ -85,7 +85,7 @@ def Preprocess(
     rotation_center = float(rotation_center)
 
     s3 = S3(num_connections=uploader_num_connections)
-    bucket, key, qs = S3ParseUrl(rem_hdf5)
+    bucket, key, qs = S3.parseUrl(rem_hdf5)
     key = os.path.basename(key)
 
     loc_hdf5 = f"{loc}/hdf5/{key}"
@@ -110,8 +110,7 @@ def Preprocess(
     logger.info(
         f" Preprocess hdf5({rem_hdf5}) rotation-center({rotation_center}) slice-range({slice_range}) recontructions({len(s3_r)}/{tot_slices}) segmentations({len(s3_s)}/{tot_slices}) ...")
 
-    from nsdf.s3 import S3Uploader
-    uploader = S3Uploader(s3)
+    uploader = s3.createUploader()
 
     white_universal = None
     trained_model = None
@@ -426,7 +425,7 @@ def ConvertImageStackMain(workflow):
     ARGS = []
     for file in files:
 
-        bucket, key, qs = S3ParseUrl(file["url"])
+        bucket, key, qs = S3.parseUrl(file["url"])
         key = os.path.basename(key)
 
         whats = []

diff --git a/nsdf/s3/__init__.py b/nsdf/s3/__init__.py
@@ -1,7 +1,3 @@
 
-from .s3 import S3ParseUrl,S3,S3GuessEndPoint
-from .sync import S3Sync
-from .uploader import S3Uploader
-from .list_objects import ListObjects
-# from .copy_objects import CopyObjects
+from .s3 import S3
 from .main import Main
diff --git a/nsdf/s3/copy_objects.py b/nsdf/s3/copy_objects.py