Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

group post with h5path option #302

Merged
merged 2 commits into from
Jan 30, 2024
Merged

group post with h5path option #302

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
960e2de
group post with h5path option
jreadey Jan 29, 2024
3ecb972
fix spelling
jreadey Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion hsds/dset_sn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1026,7 +1026,7 @@ async def POST_Dataset(request):
msg = f"chunk_size: {chunk_size}, min: {min_chunk_size}, "
msg += f"max: {max_chunk_size}"
log.debug(msg)
# nothing to do about inefficencly small chunks, but large chunks
# nothing to do about inefficiently small chunks, but large chunks
# can be subdivided
if chunk_size < min_chunk_size:
msg = f"chunk size: {chunk_size} less than min size: "
Expand Down
94 changes: 53 additions & 41 deletions hsds/group_sn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@
from aiohttp.web_exceptions import HTTPBadRequest, HTTPForbidden, HTTPNotFound
from json import JSONDecodeError

from .util.httpUtil import http_post, getHref
from .util.httpUtil import jsonResponse
from .util.idUtil import isValidUuid, getDataNodeUrl, createObjId
from .util.httpUtil import getHref, jsonResponse, getBooleanParam
from .util.idUtil import isValidUuid
from .util.authUtil import getUserPasswordFromRequest, aclCheck
from .util.authUtil import validateUserPassword
from .util.domainUtil import getDomainFromRequest, isValidDomain
from .util.domainUtil import getBucketForDomain, getPathForDomain, verifyRoot
from .servicenode_lib import getDomainJson, getObjectJson, validateAction, deleteObj
from .servicenode_lib import getObjectIdByPath, getPathForObjectId, putHardLink
from .util.linkUtil import validateLinkName
from .servicenode_lib import getDomainJson, getObjectJson, validateAction, deleteObj, createGroup
from .servicenode_lib import getObjectIdByPath, getPathForObjectId, createGroupByPath
from . import hsds_logger as log


Expand Down Expand Up @@ -66,10 +66,9 @@ async def GET_Group(request):
if group_id:
msg += f" group_id: {group_id}"
log.info(msg)
if "include_links" in params and params["include_links"]:
include_links = True
if "include_attrs" in params and params["include_attrs"]:
include_attrs = True

include_links = getBooleanParam(params, "include_links")
include_attrs = getBooleanParam(params, "include_attrs")

username, pswd = getUserPasswordFromRequest(request)
if username is None and app["allow_noauth"]:
Expand Down Expand Up @@ -159,6 +158,7 @@ async def POST_Group(request):
"""HTTP method to create new Group object"""
log.request(request)
app = request.app
params = request.rel_url.query

username, pswd = getUserPasswordFromRequest(request)
# write actions need auth
Expand All @@ -177,13 +177,16 @@ async def POST_Group(request):
aclCheck(app, domain_json, "create", username)

verifyRoot(domain_json)
root_id = domain_json["root"]

link_id = None
link_title = None
# allow parent group creation or not
implicit = getBooleanParam(params, "implicit")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Whether or not to create intermediate groups is specified in the creation properties of a group - see docs here. Would it be possible to read this from provided creation properties instead?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about we have the rest vol inspect the property list and set the query param if it applies?


parent_id = None
h5path = None
creation_props = None

if request.has_body:

try:
body = await request.json()
except JSONDecodeError:
Expand All @@ -194,44 +197,53 @@ async def POST_Group(request):
log.info(f"POST Group body: {body}")
if body:
if "link" in body:
if "h5path" in body:
msg = "link can't be used with h5path"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
link_body = body["link"]
log.debug(f"link_body: {link_body}")
if "id" in link_body:
link_id = link_body["id"]
parent_id = link_body["id"]
if "name" in link_body:
link_title = link_body["name"]
if link_id and link_title:
log.debug(f"link id: {link_id}")
# verify that the referenced id exists and is in this
# domainand that the requestor has permissions to create
# a link
act = "create"
await validateAction(app, domain, link_id, username, act)
if not link_id or not link_title:
log.warn(f"POST Group body with no link: {body}")
try:
# will throw exception if there's a slash in the name
validateLinkName(link_title)
except ValueError:
msg = f"invalid link title: {link_title}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)

if parent_id and link_title:
log.debug(f"parent id: {parent_id}, link_title: {link_title}")
h5path = link_title # just use the link name as the h5path

if "h5path" in body:
h5path = body["h5path"]
if "parent_id" not in body:
parent_id = root_id
else:
parent_id = body["parent_id"]
if "creationProperties" in body:
creation_props = body["creationProperties"]

# get again in case cache was invalidated
domain_json = await getDomainJson(app, domain)
if parent_id:
kwargs = {"bucket": bucket, "parent_id": parent_id, "h5path": h5path}
if creation_props:
kwargs["creation_props"] = creation_props
if implicit:
kwargs["implicit"] = True
log.debug(f"createGroupByPath args: {kwargs}")
group_json = await createGroupByPath(app, **kwargs)
else:
# create an anonymous group
kwargs = {"bucket": bucket, "root_id": root_id}
if creation_props:
kwargs["creation_props"] = creation_props
group_json = await createGroup(app, **kwargs)

root_id = domain_json["root"]
group_id = createObjId("groups", rootid=root_id)
log.info(f"new group id: {group_id}")
group_json = {"id": group_id, "root": root_id}
if creation_props:
group_json["creationProperties"] = creation_props
log.debug(f"create group, body: {group_json}")
req = getDataNodeUrl(app, group_id) + "/groups"
params = {"bucket": bucket}

group_json = await http_post(app, req, data=group_json, params=params)

# create link if requested
if link_id and link_title:
await putHardLink(app, link_id, link_title, tgt_id=group_id, bucket=bucket)

log.debug("returning resp")
log.debug(f"returning resp: {group_json}")
# group creation successful
resp = await jsonResponse(request, group_json, status=201)
log.response(request, resp=resp)
Expand Down
122 changes: 120 additions & 2 deletions hsds/servicenode_lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@
import asyncio
import json

from aiohttp.web_exceptions import HTTPBadRequest, HTTPForbidden
from aiohttp.web_exceptions import HTTPBadRequest, HTTPForbidden, HTTPGone, HTTPConflict
from aiohttp.web_exceptions import HTTPNotFound, HTTPInternalServerError
from aiohttp.client_exceptions import ClientOSError, ClientError

from .util.authUtil import getAclKeys
from .util.arrayUtil import encodeData
from .util.idUtil import getDataNodeUrl, getCollectionForId
from .util.idUtil import getDataNodeUrl, getCollectionForId, createObjId, getRootObjId
from .util.idUtil import isSchema2Id, getS3Key, isValidUuid
from .util.linkUtil import h5Join, validateLinkName, getLinkClass
from .util.storUtil import getStorJSONObj, isStorObj
Expand Down Expand Up @@ -465,6 +465,7 @@ async def putHardLink(app, group_id, title, tgt_id=None, bucket=None):
""" create a new hard link. Return 201 if this is a new link,
or 200 if it's a duplicate of an existing link """

log.debug(f"putHardLink for group {group_id}, tgt: {tgt_id} title: {title}")
status = await putLink(app, group_id, title, tgt_id=tgt_id, bucket=bucket)
return status

Expand All @@ -473,6 +474,7 @@ async def putSoftLink(app, group_id, title, h5path=None, bucket=None):
""" create a new soft link. Return 201 if this is a new link,
or 200 if it's a duplicate of an existing link """

log.debug(f"putSoftLink for group {group_id}, h5path: {h5path} title: {title}")
status = await putLink(app, group_id, title, h5path=h5path, bucket=bucket)
return status

Expand All @@ -481,6 +483,9 @@ async def putExternalLink(app, group_id, title, h5path=None, h5domain=None, buck
""" create a new external link. Return 201 if this is a new link,
or 200 if it's a duplicate of an existing link """

msg = f"putExternalLink for group {group_id}, "
msg += f"h5path: {h5path}, h5domain: {h5domain}"
log.debug(msg)
status = await putLink(app, group_id, title, h5path=h5path, h5domain=h5domain, bucket=bucket)
return status

Expand Down Expand Up @@ -1006,3 +1011,116 @@ async def deleteObj(app, obj_id, bucket=None):
meta_cache = app["meta_cache"]
if obj_id in meta_cache:
del meta_cache[obj_id] # remove from cache


async def createGroup(app, root_id=None, creation_props=None, bucket=None):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since the parent group that will contain the new group doesn't have to be the root group, root_id should be renamed to parent_id

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This method is for anonymous group so parent I’d wouldn’t apply. The root I’d basically determines which domain the group will belong to

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ID not Id

""" create a group object and return group json """

group_id = createObjId("groups", rootid=root_id)
log.info(f"new group id: {group_id}")
group_json = {"id": group_id, "root": root_id}
if creation_props:
group_json["creationProperties"] = creation_props
log.debug(f"createGjroup, body: {group_json}")
req = getDataNodeUrl(app, group_id) + "/groups"
params = {"bucket": bucket}
group_json = await http_post(app, req, data=group_json, params=params)

return group_json


async def createGroupByPath(app,
parent_id=None,
h5path=None,
implicit=False,
creation_props=None,
bucket=None):

""" create the group at the designated path relative to the parent.
If implicit is True, make any intermidiate groups needed in the h5path. """
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

intermidiate -> intermediate

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed


if not parent_id:
msg = "no parent_id given for createGroupByPath"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
if not h5path:
msg = "no h5path given for createGroupByPath"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
log.debug(f"createGroupByPath - parent_id: {parent_id}, h5path: {h5path}")

root_id = getRootObjId(parent_id)

if h5path.startswith("/"):
if parent_id == root_id:
# just adjust the path to be relative
h5path = h5path[1:]
else:
msg = f"createGroup expecting relative h5path, but got: {h5path}"
log.warn(msg)
raise HTTPBadRequest(reason=msg)

if h5path.endswith("/"):
h5path = h5path[:-1] # makes iterating through the links a bit easier

if not h5path:
msg = "h5path for createGroupByPath invalid"
log.warn(msg)
raise HTTPBadRequest(reason=msg)

group_json = None
link_titles = h5path.split("/")
log.debug(f"link_titles: {link_titles}")
for i in range(len(link_titles)):
if i == len(link_titles) - 1:
last_link = True
else:
last_link = False
link_title = link_titles[i]
log.debug(f"createGroupByPath - processing link: {link_title}")
link_json = None
try:
link_json = await getLink(app, parent_id, link_title, bucket=bucket)
except (HTTPNotFound, HTTPGone):
pass # link doesn't exist

if link_json:
log.debug(f"link for link_title {link_title} found: {link_json}")
# if this is the last link, that's a problem
if last_link:
msg = f"object at {h5path} already exists"
log.warn(msg)
raise HTTPConflict()
# otherwise, verify that this is a hardlink
if link_json.get("class") != "H5L_TYPE_HARD":
msg = "createGroup h5path must contain only hardlinks"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
parent_id = link_json["id"]
if getCollectionForId(parent_id) != "groups":
# parent objects must be groups!
msg = f"{link_title} is not a group"
log.warn(msg)
raise HTTPBadRequest(reason=msg)
else:
log.debug(f"link: {link_title} to sub-group found")
else:
log.debug(f"link for link_title {link_title} not found")
if not last_link and not implicit:
if len(link_titles) > 1:
msg = f"createGroupByPath failed: not all groups in {h5path} exist"
else:
msg = f"createGroupByPath failed: {h5path} does not exist"
log.warn(msg)
raise HTTPNotFound(reason=msg)
# create a group
kwargs = {"bucket": bucket, "root_id": root_id}
if creation_props:
kwargs["creation_props"] = creation_props
group_json = await createGroup(app, **kwargs)
group_id = group_json["id"]
# create a link to the new group
await putHardLink(app, parent_id, link_title, tgt_id=group_id, bucket=bucket)
parent_id = group_id # new parent
log.info(f"createGroupByPath {h5path} done")
return group_json
Loading
Loading