moved prefix to aioboto options, renamed hidden to browseable

config.template.yaml

@@ -35,16 +35,16 @@ targets:
   # using the default client (currently aioboto).
   #
   - name: cosem-data-with-prefix
-    prefix: aic_desmosome-1
     options:
       bucket: janelia-cosem-datasets
+      prefix: aic_desmosome-1
 
   #
   # Exposes s3://janelia-cosem-datasets at /cosem-data-hidden
   # using the default client (currently aioboto) and makes it unbrowseable
   #
   - name: cosem-data-hidden
-    hidden: true
+    browseable: false
     options:
       bucket: janelia-cosem-datasets
 

run.py

@@ -19,5 +19,5 @@
     )
     args = argparser.parse_args()
 
-    uvicorn.run(app, host='0.0.0.0', port=args.port, workers=1)
+    uvicorn.run(app, host='0.0.0.0', port=args.port, workers=1, reload=True)
 

tests/test_serve.py

@@ -21,12 +21,14 @@
     ),
     Target(
         name='with-prefix',
-        prefix='jrc_mus_lung_covid.n5/',
-        options={'bucket':'janelia-data-examples'}
+        options={
+            'bucket':'janelia-data-examples',
+            'prefix':'jrc_mus_lung_covid.n5/'
+        }
     ),
     Target(
         name='hidden-with-endpoint',
-        hidden=True,
+        browseable=False,
         options={
             'bucket':'janelia-data-examples',
             'endpoint':'https://s3.amazonaws.com',
@@ -48,11 +50,12 @@ def test_get_html_root():
         response = client.get("/")
         assert response.status_code == 200
         assert response.headers['content-type'].startswith("text/html")
+        logger.info(response.text)
         for target in settings.targets:
-            if target.hidden:
-                assert target.name not in response.text
-            else:
+            if target.browseable:
                 assert target.name in response.text
+            else:
+                assert target.name not in response.text
 
 
 def test_get_html_listing():

x2s3/app.py

@@ -60,7 +60,6 @@ async def startup_event():
         target_config = app.settings.get_target_config(target_key)
         proxy_kwargs = {
             'target_name': target_name,
-            'prefix': target_config.prefix
         }
 
         client = registry.client(target_config.client,
@@ -171,10 +170,7 @@ async def browse_bucket(request: Request,
         next_ct_elem = root.find('NextContinuationToken')
         next_token = next_ct_elem.text
 
-    target_prefix = ''
-    if not is_virtual:
-        target_prefix = '/'+target_name
-
+    target_prefix = '' if is_virtual else '/'+target_name
     parent_prefix = dir_path(os.path.dirname(prefix.rstrip('/')))
 
     return templates.TemplateResponse("browse.html", {
@@ -218,7 +214,7 @@ async def target_dispatcher(request: Request,
 
     if not target_name or (is_virtual and target_name=='www'):
         # Return target index
-        bucket_list = { target: f"/{target}/" for target in app.settings.get_targets()}
+        bucket_list = { target: f"/{target}/" for target in app.settings.get_browseable_targets()}
         return templates.TemplateResponse("index.html", {"request": request, "links": bucket_list})
 
     target_config = app.settings.get_target_config(target_name)
@@ -267,14 +263,8 @@ async def head_object(request: Request, path: str):
         client = get_client(target_name)
         if client is None:
             raise HTTPException(status_code=500, detail="Client for target bucket not found")
-
-        client_prefix = target_config.prefix
-
-        key = target_path
-        if client_prefix:
-            key = os.path.join(client_prefix, key) if key else client_prefix
 
-        return await client.head_object(key)
+        return await client.head_object(target_path)
     except:
         logger.opt(exception=sys.exc_info()).info("Error requesting head")
         return JSONResponse({"error":"Error requesting HEAD"}, status_code=500)

x2s3/client_aioboto.py

@@ -42,8 +42,8 @@ def __init__(self, proxy_kwargs, **kwargs):
 
         self.proxy_kwargs = proxy_kwargs or {}
         self.target_name = self.proxy_kwargs['target_name']
-        self.target_prefix = self.proxy_kwargs.get('prefix')
         self.bucket_name = kwargs['bucket']
+        self.bucket_prefix = kwargs.get('prefix')
 
         self.anonymous = True
         access_key,secret_key = '',''
@@ -76,16 +76,20 @@ def get_client_creator(self):
 
     @override
     async def head_object(self, key: str):
+        real_key = key
+        if self.bucket_prefix:
+            real_key = os.path.join(self.bucket_prefix, key) if key else self.bucket_prefix
+
         async with self.get_client_creator() as client:
             try:
-                s3_res = await client.head_object(Bucket=self.bucket_name, Key=key)
+                s3_res = await client.head_object(Bucket=self.bucket_name, Key=real_key)
                 headers = {
                     "ETag": s3_res.get("ETag"),
                     "Content-Length": str(s3_res.get("ContentLength")),
                     "Last-Modified": s3_res.get("LastModified").strftime("%a, %d %b %Y %H:%M:%S GMT")
                 }
 
-                content_type = guess_content_type(key)
+                content_type = guess_content_type(real_key)
                 headers['Content-Type'] = content_type
 
                 return Response(headers=headers)
@@ -95,10 +99,11 @@ async def head_object(self, key: str):
 
     @override
     async def get_object(self, key: str):
-        if self.target_prefix:
-            key = os.path.join(self.target_prefix, key) if key else self.target_prefix
+        real_key = key
+        if self.bucket_prefix:
+            real_key = os.path.join(self.bucket_prefix, key) if key else self.bucket_prefix
 
-        filename = os.path.basename(key)
+        filename = os.path.basename(real_key)
         headers = {}
 
         content_type = guess_content_type(filename)
@@ -111,6 +116,7 @@ async def get_object(self, key: str):
                 self.get_client_creator,
                 bucket=self.bucket_name,
                 key=key,
+                real_key=real_key,
                 media_type=content_type,
                 headers=headers)
         except Exception as e:
@@ -129,8 +135,8 @@ async def list_objects_v2(self,
 
         # prefix user-supplied prefix with configured prefix
         real_prefix = prefix
-        if self.target_prefix:
-            real_prefix = os.path.join(self.target_prefix, prefix) if prefix else self.target_prefix
+        if self.bucket_prefix:
+            real_prefix = os.path.join(self.bucket_prefix, prefix) if prefix else self.bucket_prefix
 
         # ensure the prefix ends with a slash
         if real_prefix and not real_prefix.endswith('/'):
@@ -152,13 +158,13 @@ async def list_objects_v2(self,
                 params = {k: v for k, v in params.items() if v is not None}
 
                 response = await client.list_objects_v2(**params)
-                next_token = remove_prefix(self.target_prefix, response.get("NextContinuationToken", ""))
+                next_token = remove_prefix(self.bucket_prefix, response.get("NextContinuationToken", ""))
                 is_truncated = "true" if response.get("IsTruncated", False) else "false"
 
                 contents = []
                 for obj in response.get("Contents", []):
                     contents.append({
-                        'Key': remove_prefix(self.target_prefix, obj["Key"]),
+                        'Key': remove_prefix(self.bucket_prefix, obj["Key"]),
                         'LastModified': obj["LastModified"].isoformat(),
                         'ETag': obj.get("ETag"),
                         'Size': obj.get("Size"),
@@ -167,7 +173,7 @@ async def list_objects_v2(self,
 
                 common_prefixes = []
                 for cp in response.get("CommonPrefixes", []):
-                    common_prefix = remove_prefix(self.target_prefix, cp["Prefix"])
+                    common_prefix = remove_prefix(self.bucket_prefix, cp["Prefix"])
                     common_prefixes.append(common_prefix)
 
                 kwargs = {
@@ -205,17 +211,19 @@ def __init__(
             media_type: str = None,
             background: BackgroundTask = None,
             bucket: str = None,
-            key: str = None
+            key: str = None,
+            real_key: str = None,
     ):
         super(S3Stream, self).__init__(content, status_code, headers, media_type, background)
         self.client_creator = client_creator
         self.bucket = bucket
         self.key = key
+        self.real_key = real_key
 
     async def stream_response(self, send) -> None:
         async with self.client_creator() as client:
             try:
-                result = await client.get_object(Bucket=self.bucket, Key=self.key)
+                result = await client.get_object(Bucket=self.bucket, Key=self.real_key)
 
                 await send({
                     "type": "http.response.start",

x2s3/client_file.py

@@ -32,8 +32,8 @@ class FileProxyClient(ProxyClient):
     def __init__(self, proxy_kwargs, **kwargs):
         self.proxy_kwargs = proxy_kwargs or {}
         self.target_name = self.proxy_kwargs['target_name']
-        self.target_prefix = self.proxy_kwargs.get('prefix')
-        self.root_path = str(Path(kwargs['root']).resolve())
+        self.target_prefix = kwargs.get('prefix') # Not very useful and may be removed in the future
+        self.root_path = str(Path(kwargs['path']).resolve())
 
     @override
     async def head_object(self, key: str):
@@ -89,7 +89,6 @@ async def get_object(self, key: str):
             return handle_exception(e, key)
 
 
-
     @override
     async def list_objects_v2(self,
                             continuation_token: str,

x2s3/settings.py

@@ -11,8 +11,7 @@
 
 class Target(BaseModel):
     name: str
-    hidden: bool = False
-    prefix: str = None
+    browseable: bool = True
     client: str = "aioboto"
     options: Dict[str,str] = {}
 
@@ -42,8 +41,8 @@ def __init__(self, **data) -> None:
         self.target_map = {t.name.lower(): t for t in self.targets}
 
 
-    def get_targets(self):
-        return [k for k in self.target_map if not self.target_map[k].hidden]
+    def get_browseable_targets(self):
+        return [target.name for target in self.targets if target.browseable]
 
 
     def get_target_config(self, name):