From 8e9ebf9eb061dd17f910747564bfb7e9c9fbe244 Mon Sep 17 00:00:00 2001 From: KrKOo Date: Fri, 10 Nov 2023 15:23:54 +0100 Subject: [PATCH] Init chart and exchange --- nbgrader-chart/.gitignore | 1 + nbgrader-chart/.helmignore | 23 + nbgrader-chart/Chart.lock | 6 + nbgrader-chart/Chart.yaml | 11 + nbgrader-chart/templates/_helpers.tpl | 17 + nbgrader-chart/templates/cm.yaml | 34 ++ nbgrader-chart/templates/pvc.yaml | 12 + nbgrader-chart/templates/svc.yaml | 9 + nbgrader-chart/values.yaml | 519 ++++++++++++++++++ nbgrader-exchange/.gitignore | 2 + nbgrader-exchange/README.md | 1 + .../nbgrader_k8s_exchange/__init__.py | 0 .../nbgrader_k8s_exchange/plugin/__init__.py | 20 + .../nbgrader_k8s_exchange/plugin/collect.py | 143 +++++ .../nbgrader_k8s_exchange/plugin/exchange.py | 156 ++++++ .../plugin/fetch_assignment.py | 83 +++ .../plugin/fetch_feedback.py | 111 ++++ .../nbgrader_k8s_exchange/plugin/list.py | 241 ++++++++ .../plugin/release_assignment.py | 113 ++++ .../plugin/release_feedback.py | 92 ++++ .../nbgrader_k8s_exchange/plugin/submit.py | 160 ++++++ nbgrader-exchange/pyproject.toml | 26 + 22 files changed, 1780 insertions(+) create mode 100644 nbgrader-chart/.gitignore create mode 100644 nbgrader-chart/.helmignore create mode 100644 nbgrader-chart/Chart.lock create mode 100644 nbgrader-chart/Chart.yaml create mode 100644 nbgrader-chart/templates/_helpers.tpl create mode 100644 nbgrader-chart/templates/cm.yaml create mode 100644 nbgrader-chart/templates/pvc.yaml create mode 100644 nbgrader-chart/templates/svc.yaml create mode 100644 nbgrader-chart/values.yaml create mode 100644 nbgrader-exchange/.gitignore create mode 100644 nbgrader-exchange/README.md create mode 100644 nbgrader-exchange/nbgrader_k8s_exchange/__init__.py create mode 100644 nbgrader-exchange/nbgrader_k8s_exchange/plugin/__init__.py create mode 100644 nbgrader-exchange/nbgrader_k8s_exchange/plugin/collect.py create mode 100644 nbgrader-exchange/nbgrader_k8s_exchange/plugin/exchange.py create mode 100644 nbgrader-exchange/nbgrader_k8s_exchange/plugin/fetch_assignment.py create mode 100644 nbgrader-exchange/nbgrader_k8s_exchange/plugin/fetch_feedback.py create mode 100644 nbgrader-exchange/nbgrader_k8s_exchange/plugin/list.py create mode 100644 nbgrader-exchange/nbgrader_k8s_exchange/plugin/release_assignment.py create mode 100644 nbgrader-exchange/nbgrader_k8s_exchange/plugin/release_feedback.py create mode 100644 nbgrader-exchange/nbgrader_k8s_exchange/plugin/submit.py create mode 100644 nbgrader-exchange/pyproject.toml diff --git a/nbgrader-chart/.gitignore b/nbgrader-chart/.gitignore new file mode 100644 index 0000000..25b4569 --- /dev/null +++ b/nbgrader-chart/.gitignore @@ -0,0 +1 @@ +**/charts/*.tgz diff --git a/nbgrader-chart/.helmignore b/nbgrader-chart/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/nbgrader-chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/nbgrader-chart/Chart.lock b/nbgrader-chart/Chart.lock new file mode 100644 index 0000000..9045fb5 --- /dev/null +++ b/nbgrader-chart/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: jupyterhub + repository: https://hub.jupyter.org/helm-chart/ + version: 3.0.3 +digest: sha256:fad52e3374588bd08d3382acd452a9335ad66337b49ce973d1b6383c185d4388 +generated: "2023-11-10T10:37:31.611068395+01:00" diff --git a/nbgrader-chart/Chart.yaml b/nbgrader-chart/Chart.yaml new file mode 100644 index 0000000..32efe26 --- /dev/null +++ b/nbgrader-chart/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: nbgrader +description: A Helm chart for deploying nbgrader +type: application +version: 0.1.0 +appVersion: '0.9.1' + +dependencies: + - name: jupyterhub + version: 3.0.3 + repository: https://hub.jupyter.org/helm-chart/ diff --git a/nbgrader-chart/templates/_helpers.tpl b/nbgrader-chart/templates/_helpers.tpl new file mode 100644 index 0000000..0c2cfec --- /dev/null +++ b/nbgrader-chart/templates/_helpers.tpl @@ -0,0 +1,17 @@ +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "nbgrader.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} diff --git a/nbgrader-chart/templates/cm.yaml b/nbgrader-chart/templates/cm.yaml new file mode 100644 index 0000000..dc28236 --- /dev/null +++ b/nbgrader-chart/templates/cm.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nbgrader-config-global +data: + nbgrader_config.py: | + import os + from nbgrader.server_extensions.assignment_list.handlers import AssignmentList + from nbgrader.auth import Authenticator, JupyterHubAuthPlugin + + c = get_config() + c.Exchange.path_includes_course = True + c.Exchange.root = "/mnt/exchange" + c.Authenticator.plugin_class = JupyterHubAuthPlugin + + c.ExchangeFactory.collect = 'nbgrader_k8s_exchange.plugin.ExchangeCollect' + c.ExchangeFactory.exchange = 'nbgrader_k8s_exchange.plugin.Exchange' + c.ExchangeFactory.fetch_assignment = 'nbgrader_k8s_exchange.plugin.ExchangeFetchAssignment' + c.ExchangeFactory.fetch_feedback = 'nbgrader_k8s_exchange.plugin.ExchangeFetchFeedback' + c.ExchangeFactory.list = 'nbgrader_k8s_exchange.plugin.ExchangeList' + c.ExchangeFactory.release_assignment = 'nbgrader_k8s_exchange.plugin.ExchangeReleaseAssignment' + c.ExchangeFactory.release_feedback = 'nbgrader_k8s_exchange.plugin.ExchangeReleaseFeedback' + c.ExchangeFactory.submit = 'nbgrader_k8s_exchange.plugin.ExchangeSubmit' + + # List courses for the student even if there are no assignments + def list_courses(self): + auth = Authenticator(config=c) + + return { + "success": True, + "value": auth.get_student_courses(os.environ['JUPYTERHUB_USER']) + } + + AssignmentList.list_courses = list_courses diff --git a/nbgrader-chart/templates/pvc.yaml b/nbgrader-chart/templates/pvc.yaml new file mode 100644 index 0000000..345b4c5 --- /dev/null +++ b/nbgrader-chart/templates/pvc.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: nbgrader-exchange +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: {{ .Values.exchange.size }} + storageClassName: {{ .Values.exchange.storageClassName }} + volumeMode: Filesystem diff --git a/nbgrader-chart/templates/svc.yaml b/nbgrader-chart/templates/svc.yaml new file mode 100644 index 0000000..2bf6c38 --- /dev/null +++ b/nbgrader-chart/templates/svc.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nbgrader.fullname" . }} +spec: + clusterIP: None + selector: + app: jupyterhub + component: hub diff --git a/nbgrader-chart/values.yaml b/nbgrader-chart/values.yaml new file mode 100644 index 0000000..c899074 --- /dev/null +++ b/nbgrader-chart/values.yaml @@ -0,0 +1,519 @@ +exchange: + # Select the size of the exchange volume + size: 10Gi + # Select the storage class + storageClassName: nfs-csi + +# https://github.com/jupyterhub/zero-to-jupyterhub-k8s/blob/main/jupyterhub/values.yaml +jupyterhub: + debug: + enabled: true + + proxy: + service: + type: ClusterIP + chp: + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: 1000m + memory: 512Mi + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + extraPodSpec: + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + + networkPolicy: + enabled: false + + rbac: + create: true + + ingress: + ## Example ingress configuration + # enabled: true + # hosts: + # - nbgrader.example.com + # tls: + # - hosts: + # - nbgrader.example.com + # secretName: nbgrader-example-com-tls + + enabled: false + annotations: {} + hosts: [] + tls: [] + + hub: + extraConfig: + 01-extra-config: | + import os, nativeauthenticator + + c.MappingKernelManager.cull_connected = False + c.MappingKernelManager.cull_busy = False + c.MappingKernelManager.cull_idle_timeout = 259200 + c.NotebookApp.shutdown_no_activity_timeout = 259200 + + c.KubeSpawner.auth_state_hook = userdata_hook + c.KubeSpawner.pre_spawn_hook = bootstrap_pre_spawn + c.KubeSpawner.automount_service_account_token = False + + c.JupyterHub.template_paths = ['/etc/jupyterhub/templates/', f"{os.path.dirname(nativeauthenticator.__file__)}/templates/"] + + 02-extra-config: | + from jupyterhub.handlers import BaseHandler + from nativeauthenticator.handlers import SignUpHandler + from nativeauthenticator import NativeAuthenticator + + # In the /hub/home "Services" dropdown, show only courses that the user is enrolled in + def get_accessible_services(self, user): + accessible_services = [] + if user is None: + return accessible_services + + courses = [] + for group in user.groups: + if group.name.startswith("formgrade-"): + courses.append(group.name.replace("formgrade-", "")) + + for service in self.services.values(): + if not service.name in courses: + continue + if not service.url: + continue + if not service.display: + continue + accessible_services.append(service) + + return accessible_services + + # Allow only alphanumeric usernames + def make_validate_username(cls): + __class__ = cls + def validate_username(self, username): + if not username.isalnum(): + return False + if not username[0].isalpha(): + return False + return super().validate_username(username) + return validate_username + + # Update sign up error messages + def get_result_message( + self, + user, + assume_user_is_human, + username_already_taken, + confirmation_matches, + user_is_admin, + ): + if not assume_user_is_human: + alert = "alert-danger" + message = "You failed the reCAPTCHA. Please try again" + elif username_already_taken: + alert = "alert-danger" + message = ( + "Something went wrong!\nIt appears that this " + "username is already in use. Please try again " + "with a different username." + ) + elif not confirmation_matches: + alert = "alert-danger" + message = "Your password did not match the confirmation. Please try again." + elif not user: + alert = "alert-danger" + minimum_password_length = self.authenticator.minimum_password_length + if minimum_password_length > 0: + message = ( + "Something went wrong!\nBe sure your username " + "does only contain alphanumeric characters, your " + f"password has at least {minimum_password_length} " + "characters and is not too common." + ) + else: + message = ( + "Something went wrong!\nBe sure your username " + "does only contain alphanumeric characters and your " + "password is not too common." + ) + # If user creation went through & open-signup is enabled, success. + # If user creation went through & the user is an admin, also success. + elif (user is not None) and (self.authenticator.open_signup or user_is_admin): + alert = "alert-success" + message = ( + "The signup was successful! You can now go to " + "the home page and log in to the system." + ) + else: + # Default response if nothing goes wrong. + alert = "alert-info" + message = "Your information has been sent to the admin." + + if (user is not None) and user.login_email_sent: + message = ( + "The signup was successful! Check your email " + "to authorize your access." + ) + + return alert, message + + BaseHandler.get_accessible_services = get_accessible_services + NativeAuthenticator.validate_username = make_validate_username(NativeAuthenticator) + SignUpHandler.get_result_message = get_result_message + + 00-extra-config: | + from traitlets import default, Unicode + from tornado import gen + from kubespawner import KubeSpawner + import asyncio + import kubernetes_asyncio + from kubernetes_asyncio import config, client + from kubernetes_asyncio.client import ( + V1ObjectMeta, + V1Secret, + V1PersistentVolume, + V1PersistentVolumeClaim, + V1ResourceRequirements, + V1LabelSelector, + V1CSIPersistentVolumeSource, + V1PersistentVolumeSpec, + V1PersistentVolumeClaimSpec, + ApiException, + ) + + # Define all the courses and their instructors + COURSES={"course1":["instructor1", "instructor2"], "course2":["instructor1"], "course3": ["instructor2"]} + + def userdata_hook(spawner, auth_state): + spawner.userdata = auth_state + + async def check_pvc(home_pvc_name, namespace): + async with kubernetes_asyncio.client.ApiClient() as api_client: + v1 = kubernetes_asyncio.client.CoreV1Api(api_client) + pvcs = await v1.list_namespaced_persistent_volume_claim(namespace) + for claim in pvcs.items: + if claim.metadata.name == home_pvc_name: + return claim + return None + + async def delete_pvc(namespace, pvc): + async with kubernetes_asyncio.client.ApiClient() as api_client: + v1 = kubernetes_asyncio.client.CoreV1Api(api_client) + await v1.delete_namespaced_persistent_volume_claim(name=pvc, namespace=namespace) + await asyncio.sleep(1) + + async def create_pvc(home_pvc_name, home_pv_name, namespace, storage_class, capacity): + pvc = V1PersistentVolumeClaim() + pvc.api_version = "v1" + pvc.kind = "PersistentVolumeClaim" + pvc.metadata = V1ObjectMeta() + pvc.metadata.name = home_pvc_name + pvc.spec = V1PersistentVolumeClaimSpec() + pvc.spec.access_modes = ['ReadWriteMany'] + pvc.spec.resources = V1ResourceRequirements() + pvc.spec.resources.requests = {"storage": capacity} + pvc.spec.storage_class_name = storage_class + if storage_class != "nfs-csi": + pvc.spec.selector = V1LabelSelector() + pvc.spec.selector.match_labels = {"name": home_pv_name} + try: + async with kubernetes_asyncio.client.ApiClient() as api_client: + v1 = kubernetes_asyncio.client.CoreV1Api(api_client) + x = await v1.create_namespaced_persistent_volume_claim(namespace, pvc) + await asyncio.sleep(1) + except ApiException as e: + if re.search("object is being deleted:", e.body): + raise web.HTTPError(401, "Can't delete PVC {}, please contact administrator!".format(home_pvc_name)) + return False + return True + + def add_volume(spawner_vol_list, volume, volname): + if len(spawner_vol_list) == 0: + spawner_vol_list = [volume] + else: + volume_exists = False + for vol in spawner_vol_list: + if "name" in vol and vol["name"] == volname: + volume_exists = True + if not volume_exists: + spawner_vol_list.append(volume) + + def mount(spawner, pv, pvc, mountpath, type): + volume = {} + volume_mount = {} + if type == "pvc": + volume = {"name": pv, "persistentVolumeClaim": {"claimName": pvc}} + volume_mount = {"mountPath": mountpath, "name": pv} + elif type == "cm": + volume = {"name": pv, "configMap": {"name": pvc}} + volume_mount = {"mountPath": mountpath, "name": pv} + add_volume(spawner.volumes, volume, pv) + add_volume(spawner.volume_mounts, volume_mount, pvc) + + async def mount_persistent_hub_home(spawner, username, namespace): + hub_home_name = username + "-home-default" + + pvc = await check_pvc(hub_home_name, namespace) + if not pvc: + await create_pvc(hub_home_name, hub_home_name + "-pv", namespace, "nfs-csi", "10Gi") + + mount(spawner, hub_home_name + "-pv", hub_home_name, "/home/jovyan", "pvc") + + def set_resources(spawner): + spawner.cpu_limit = 2 + spawner.cpu_guarantee = 2 + spawner.mem_limit = '4G' + spawner.mem_guarantee = '4G' + spawner.container_security_context = {"capabilities": {"drop": ["ALL"]}} + + + async def bootstrap_pre_spawn(spawner): + config.load_incluster_config() + groups = [] + for n in spawner.user.groups: + groups.append(n.name) + namespace = spawner.namespace + username = spawner.user.name + + spawner.start_timeout = 600 + + await mount_persistent_hub_home(spawner, username, namespace) + + isStudent = True + volume = {"name": "nbgrader-exchange", "persistentVolumeClaim": {"claimName": "nbgrader-exchange"}} + add_volume(spawner.volumes, volume, volume["name"]) + + for group in groups: + if group.startswith("formgrade-"): + isStudent = False + courseName = group.replace("formgrade-", "") + + volume_mount = {"mountPath": f"/mnt/exchange", "name": "nbgrader-exchange"} + add_volume(spawner.volume_mounts, volume_mount, volume_mount["name"]) + elif group.startswith("nbgrader-"): + courseName = group.replace("nbgrader-", "") + + if not os.path.exists(f'/mnt/exchange/{courseName}/inbound/{username}'): + os.makedirs(f"/mnt/exchange/{courseName}/inbound/{username}") + if not os.path.exists(f'/mnt/exchange/{courseName}/feedback_public/{username}'): + os.makedirs(f"/mnt/exchange/{courseName}/feedback_public/{username}") + + spawner.volume_mounts.extend([{ + "name": "nbgrader-exchange", + "mountPath": f"/mnt/exchange/{courseName}/inbound/{username}", + "subPath": f"{courseName}/inbound/{username}", + }, + { + "name": "nbgrader-exchange", + "mountPath": f"/mnt/exchange/{courseName}/feedback_public/{username}", + "subPath": f"{courseName}/feedback_public/{username}", + }, + { + "name": "nbgrader-exchange", + "mountPath": f"/mnt/exchange/{courseName}/outbound", + "subPath": f"{courseName}/outbound", + "readOnly": True + } + ]) + + if isStudent: + spawner.image = "cerit.io/hubs/nbgrader-student:10-11-2023" + else: + spawner.image = "cerit.io/hubs/nbgrader-instructor:10-11-2023" + + set_resources(spawner) + + if "--SingleUserNotebookApp.max_body_size=6291456000" not in spawner.args: + spawner.args.append("--SingleUserNotebookApp.max_body_size=6291456000") + + + groupsToCreate = {} + + base_port = 9000 + idx = 0 + for course, instructors in COURSES.items(): + # "outbound" must exist before starting a singleuser + if not os.path.exists(f'/mnt/exchange/{course}/outbound'): + os.makedirs(f"/mnt/exchange/{course}/outbound") + if not os.path.exists(f'/mnt/exchange/{course}/inbound'): + os.makedirs(f"/mnt/exchange/{course}/inbound") + if not os.path.exists(f'/mnt/exchange/{course}/feedback_public'): + os.makedirs(f"/mnt/exchange/{course}/feedback_public") + + with open(f"/mnt/exchange/{course}/nbgrader_config.py", "w") as f: + f.write("c = get_config()\n") + f.write(f"c.CourseDirectory.root = '/mnt/exchange/{course}'\n") + f.write(f"c.CourseDirectory.course_id = '{course}'\n") + + c.JupyterHub.services.append( + { + "name": course, + "url": f"http://course-svc:{base_port + idx}", + "command": ["jupyterhub-singleuser", f"--group=formgrade-{course}", f"--port={base_port + idx}", "--debug", "--ServerApp.ip=0.0.0.0"], + "cwd": f"/mnt/exchange/{course}", + "oauth_no_confirm": True, + "environment" : { + # Here nbgrader.auth.JupyterHubAuthPlugin needs a user, that always exists + "JUPYTERHUB_USER": "admin" + } + } + ) + + groupsToCreate[f"nbgrader-{course}"] = [] + groupsToCreate[f"formgrade-{course}"] = instructors + + c.JupyterHub.load_roles.append({ + "name": f"formgrade-{course}", + "groups": [f"formgrade-{course}"], + "services": [course], + "scopes": [ + f"access:services!service={course}", + f"read:services!service={course}", + f"list:services!service={course}", + "groups", + "users" + ] + }) + idx += 1 + + c.JupyterHub.load_groups = groupsToCreate + + config: + Authenticator: + # If any more admins, existing ones must set as admin in UI + # Changes will disappear if db disappears + admin_users: + - 'admin' + enable_auth_state: true + auto_login: False + NativeAuthenticator: + check_common_password: true + minimum_password_length: 8 + ask_email_on_signup: true + JupyterHub: + authenticator_class: 'native' + # Allow servers do anything their owners can do + load_roles: + - name: 'server' + scopes: + - 'inherit' + image: + name: cerit.io/hubs/nbgrader-hub + tag: '10-11-2023' + pullPolicy: Always + resources: + requests: + memory: '4Gi' + cpu: '2000m' + limits: + memory: '4Gi' + cpu: '2000m' + livenessProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 10 + timeoutSeconds: 10 + readinessProbe: + initialDelaySeconds: 10 + periodSeconds: 10 + failureThreshold: 10 + timeoutSeconds: 10 + db: + # Select the storage class for the hub database + pvc: + storageClassName: 'nfs-csi' + containerSecurityContext: + allowPrivilegeEscalation: false + runAsUser: 1000 + runAsGroup: 1000 + capabilities: + drop: + - ALL + podSecurityContext: + fsGroup: 2000 + fsGroupChangePolicy: OnRootMismatch + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + consecutiveFailureLimit: 0 + networkPolicy: + enabled: false + interNamespaceAccessLabels: 'accept' + egressAllowRules: + cloudMetadataServer: false + # Exchange dir mounted to hub pod to ensure student path creation + extraVolumes: + - name: nbgrader-exchange + persistentVolumeClaim: + claimName: nbgrader-exchange + - name: nbgrader-config-global + configMap: + name: nbgrader-config-global + extraVolumeMounts: + - name: nbgrader-exchange + mountPath: '/mnt/exchange' + - name: nbgrader-config-global + mountPath: '/etc/jupyter/' + readOnly: true + + singleuser: + networkPolicy: + enabled: false + cloudMetadata: + blockWithIptables: false + defaultUrl: '/lab' + storage: + type: none + extraVolumes: + - name: nbgrader-config-global + configMap: + name: nbgrader-config-global + extraVolumeMounts: + - name: nbgrader-config-global + mountPath: '/etc/jupyter/' + readOnly: true + cmd: jupyterhub-singleuser + uid: 1000 + fsGid: 100 + startTimeout: 300 + allowPrivilegeEscalation: false + extraPodConfig: + securityContext: + fsGroup: 100 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + lifecycleHooks: + postStart: + exec: + command: + - 'bash' + - '-c' + - > + echo -e "envs_dirs:\n - /home/jovyan/my-conda-envs/" > /home/jovyan/.condarc; + + scheduling: + userScheduler: + enabled: false + userPlaceholder: + enabled: false + + prePuller: + hook: + enabled: false + continuous: + enabled: false + + cull: + enabled: false + users: true + timeout: 259200 + every: 3600 diff --git a/nbgrader-exchange/.gitignore b/nbgrader-exchange/.gitignore new file mode 100644 index 0000000..9cb9928 --- /dev/null +++ b/nbgrader-exchange/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +dist/ \ No newline at end of file diff --git a/nbgrader-exchange/README.md b/nbgrader-exchange/README.md new file mode 100644 index 0000000..bf683cc --- /dev/null +++ b/nbgrader-exchange/README.md @@ -0,0 +1 @@ +# Nbgrader K8s Exchange diff --git a/nbgrader-exchange/nbgrader_k8s_exchange/__init__.py b/nbgrader-exchange/nbgrader_k8s_exchange/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nbgrader-exchange/nbgrader_k8s_exchange/plugin/__init__.py b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/__init__.py new file mode 100644 index 0000000..13590d3 --- /dev/null +++ b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/__init__.py @@ -0,0 +1,20 @@ +from .exchange import ExchangeError, Exchange +from .submit import ExchangeSubmit +from .release_feedback import ExchangeReleaseFeedback +from .release_assignment import ExchangeReleaseAssignment +from .fetch_feedback import ExchangeFetchFeedback +from .fetch_assignment import ExchangeFetchAssignment +from .collect import ExchangeCollect +from .list import ExchangeList + +__all__ = [ + "Exchange", + "ExchangeError", + "ExchangeCollect", + "ExchangeFetchAssignment", + "ExchangeFetchFeedback", + "ExchangeList", + "ExchangeReleaseAssignment", + "ExchangeReleaseFeedback", + "ExchangeSubmit" +] diff --git a/nbgrader-exchange/nbgrader_k8s_exchange/plugin/collect.py b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/collect.py new file mode 100644 index 0000000..1027805 --- /dev/null +++ b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/collect.py @@ -0,0 +1,143 @@ +import os +import glob +import shutil +import sys +from collections import defaultdict +from textwrap import dedent +import datetime + +from nbgrader.exchange.abc import ExchangeCollect as ABCExchangeCollect +from .exchange import Exchange + +from nbgrader.utils import check_mode, parse_utc +from nbgrader.api import Gradebook, MissingEntry +from nbgrader.utils import check_mode, parse_utc + +# pwd is for matching unix names with student ide, so we shouldn't import it on +# windows machines +if sys.platform != 'win32': + import pwd +else: + pwd = None + + +def groupby(l, key=lambda x: x): + d = defaultdict(list) + for item in l: + d[key(item)].append(item) + return d + + +class ExchangeCollect(Exchange, ABCExchangeCollect): + def _path_to_record(self, path): + filename = os.path.split(path)[1] + # Only split twice on +, giving three components. This allows usernames with +. + filename_list = filename.rsplit('+', 3) + if len(filename_list) < 3: + self.fail("Invalid filename: {}".format(filename)) + username = filename_list[0] + timestamp = parse_utc(filename_list[2]) + return {'username': username, 'filename': filename, 'timestamp': timestamp} + + def _sort_by_timestamp(self, records): + return sorted(records, key=lambda item: item['timestamp'], reverse=True) + + def init_src(self): + if self.coursedir.course_id == '': + self.fail("No course id specified. Re-run with --course flag.") + + self.course_path = os.path.join(self.root, self.coursedir.course_id) + self.inbound_path = os.path.join(self.course_path, 'inbound') + if not os.path.isdir(self.inbound_path): + self.fail("Course not found: {}".format(self.inbound_path)) + if not check_mode(self.inbound_path, read=True, execute=True): + self.fail("You don't have read permissions for the directory: {}".format(self.inbound_path)) + student_id = self.coursedir.student_id if self.coursedir.student_id else '*' + pattern = os.path.join(self.inbound_path, student_id, '{}+{}+*'.format(student_id, self.coursedir.assignment_id)) + records = [self._path_to_record(f) for f in glob.glob(pattern)] + usergroups = groupby(records, lambda item: item['username']) + + with Gradebook(self.coursedir.db_url, self.coursedir.course_id) as gb: + try: + assignment = gb.find_assignment(self.coursedir.assignment_id) + self.duedate = assignment.duedate + except MissingEntry: + self.duedate = None + if self.duedate is None or not self.before_duedate: + self.src_records = [self._sort_by_timestamp(v)[0] for v in usergroups.values()] + else: + self.src_records = [] + for v in usergroups.values(): + records = self._sort_by_timestamp(v) + records_before_duedate = [record for record in records if record['timestamp'] <= self.duedate] + if records_before_duedate: + self.src_records.append(records_before_duedate[0]) + else: + self.src_records.append(records[0]) + + def init_dest(self): + pass + + def copy_files(self): + if len(self.src_records) == 0: + self.log.warning("No submissions of '{}' for course '{}' to collect".format( + self.coursedir.assignment_id, + self.coursedir.course_id)) + else: + self.log.info("Processing {} submissions of '{}' for course '{}'".format( + len(self.src_records), + self.coursedir.assignment_id, + self.coursedir.course_id)) + + for rec in self.src_records: + student_id = rec['username'] + src_path = os.path.join(self.inbound_path, student_id, rec['filename']) + + # Cross check the student id with the owner of the submitted directory + if self.check_owner and pwd is not None: # check disabled under windows + try: + owner = pwd.getpwuid(os.stat(src_path).st_uid).pw_name + except KeyError: + owner = "unknown id" + if student_id != owner: + self.log.warning(dedent( + """ + {} claims to be submitted by {} but is owned by {}; cheating attempt? + you may disable this warning by unsetting the option CollectApp.check_owner + """).format(src_path, student_id, owner)) + + dest_path = self.coursedir.format_path(self.coursedir.submitted_directory, student_id, self.coursedir.assignment_id) + if not os.path.exists(os.path.dirname(dest_path)): + os.makedirs(os.path.dirname(dest_path)) + + copy = False + updating = False + if os.path.isdir(dest_path): + existing_timestamp = self.coursedir.get_existing_timestamp(dest_path) + new_timestamp = rec['timestamp'] + if self.update and (existing_timestamp is None or new_timestamp > existing_timestamp): + copy = True + updating = True + elif self.before_duedate and existing_timestamp != new_timestamp: + copy = True + updating = True + + else: + copy = True + + if copy: + if updating: + self.log.info("Updating submission: {} {}".format(student_id, self.coursedir.assignment_id)) + shutil.rmtree(dest_path) + else: + self.log.info("Collecting submission: {} {}".format(student_id, self.coursedir.assignment_id)) + self.do_copy(src_path, dest_path) + else: + if self.update: + self.log.info("No newer submission to collect: {} {}".format( + student_id, self.coursedir.assignment_id + )) + else: + self.log.info("Submission already exists, use --update to update: {} {}".format( + student_id, self.coursedir.assignment_id + )) diff --git a/nbgrader-exchange/nbgrader_k8s_exchange/plugin/exchange.py b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/exchange.py new file mode 100644 index 0000000..b91f659 --- /dev/null +++ b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/exchange.py @@ -0,0 +1,156 @@ +import os +import datetime +import sys +import shutil +import glob + +from textwrap import dedent + + +from rapidfuzz import fuzz +from traitlets import Unicode, Bool, default +from jupyter_core.paths import jupyter_data_dir + +from nbgrader.exchange.abc import Exchange as ABCExchange +from nbgrader.exchange import ExchangeError +from nbgrader.utils import check_directory, ignore_patterns, self_owned + + +class Exchange(ABCExchange): + root = Unicode( + "/usr/local/share/nbgrader/exchange", + help="The nbgrader exchange directory writable to everyone. MUST be preexisting." + ).tag(config=True) + + cache = Unicode( + "", + help="Local cache directory for nbgrader submit and nbgrader list. Defaults to $JUPYTER_DATA_DIR/nbgrader_cache" + ).tag(config=True) + + @default("cache") + def _cache_default(self): + return os.path.join(jupyter_data_dir(), 'nbgrader_cache') + + path_includes_course = Bool( + False, + help=dedent( + """ + Whether the path for fetching/submitting assignments should be + prefixed with the course name. If this is `False`, then the path + will be something like `./ps1`. If this is `True`, then the path + will be something like `./course123/ps1`. + """ + ) + ).tag(config=True) + + def set_perms(self, dest, fileperms, dirperms): + all_dirs = [] + for dirname, _, filenames in os.walk(dest): + for filename in filenames: + os.chmod(os.path.join(dirname, filename), fileperms) + all_dirs.append(dirname) + + for dirname in all_dirs[::-1]: + os.chmod(dirname, dirperms) + + def ensure_root(self): + """See if the exchange directory exists and is writable, fail if not.""" + if not check_directory(self.root, write=True, execute=True): + self.fail("Unwritable directory, please contact your instructor: {}".format(self.root)) + + def init_src(self): + """Compute and check the source paths for the transfer.""" + raise NotImplementedError + + def init_dest(self): + """Compute and check the destination paths for the transfer.""" + raise NotImplementedError + + def copy_files(self): + """Actually do the file transfer.""" + raise NotImplementedError + + def get_size(self, root): + """ + Return total size of directory in bytes. + """ + total_size = 0 + for dirpath, dirnames, filenames in os.walk(root): + for f in filenames: + fp = os.path.join(dirpath, f) + # skip if it is symbolic link + if not os.path.islink(fp): + total_size += os.path.getsize(fp) + return total_size + + + def do_copy(self, src, dest, log=None): + """ + Copy the src dir to the dest dir, omitting excluded + file/directories, non included files, and too large files, as + specified by the options coursedir.ignore, coursedir.include + and coursedir.max_file_size. + """ + dir_size = self.get_size(src) + max_dir_size = self.coursedir.max_dir_size + if dir_size > 1000 * max_dir_size: + self.log.error("Directory size is too big") + raise RuntimeError(f"Directory size is too big. Size is {dir_size}, maximum size is {1000 * max_dir_size}") + + shutil.copytree(src, dest, + ignore=ignore_patterns(exclude=self.coursedir.ignore, + include=self.coursedir.include, + max_file_size=self.coursedir.max_file_size, + log=self.log)) + # copytree copies access mode too - so we must add go+rw back to it if + # we are in groupshared. + if self.coursedir.groupshared: + for dirname, _, filenames in os.walk(dest): + # dirs become ug+rwx + st_mode = os.stat(dirname).st_mode + if st_mode & 0o2770 != 0o2770: + try: + os.chmod(dirname, (st_mode|0o2770) & 0o2777) + except PermissionError: + self.log.warning("Could not update permissions of %s to make it groupshared", dirname) + + for filename in filenames: + filename = os.path.join(dirname, filename) + st_mode = os.stat(filename).st_mode + if st_mode & 0o660 != 0o660: + try: + os.chmod(filename, (st_mode|0o660) & 0o777) + except PermissionError: + self.log.warning("Could not update permissions of %s to make it groupshared", filename) + + def start(self): + if sys.platform == 'win32': + self.fail("Sorry, the exchange is not available on Windows.") + if not self.coursedir.groupshared: + # This just makes sure that directory is o+rwx. In group shared + # case, it is up to admins to ensure that instructors can write + # there. + self.ensure_root() + + return super(Exchange, self).start() + + def _assignment_not_found(self, src_path, other_path): + msg = "Assignment not found at: {}".format(src_path) + self.log.fatal(msg) + found = glob.glob(other_path) + if found: + scores = sorted([(fuzz.ratio(self.src_path, x), x) for x in found]) + self.log.error("Did you mean: %s", scores[-1][1]) + + raise ExchangeError(msg) + + def ensure_directory(self, path, mode): + """Ensure that the path exists, has the right mode and is self owned.""" + if not os.path.isdir(path): + os.makedirs(path) + # For some reason, Python won't create a directory with a mode of 0o733 + # so we have to create and then chmod. + os.chmod(path, mode) + else: + if not self.coursedir.groupshared and not self_owned(path): + self.fail("You don't own the directory: {}".format(path)) diff --git a/nbgrader-exchange/nbgrader_k8s_exchange/plugin/fetch_assignment.py b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/fetch_assignment.py new file mode 100644 index 0000000..32bcce6 --- /dev/null +++ b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/fetch_assignment.py @@ -0,0 +1,83 @@ +import os +import shutil + +from nbgrader.exchange.abc import ExchangeFetchAssignment as ABCExchangeFetchAssignment +from nbgrader.exchange.default import Exchange +from nbgrader.utils import check_mode + + +class ExchangeFetchAssignment(Exchange, ABCExchangeFetchAssignment): + + def _load_config(self, cfg, **kwargs): + if 'ExchangeFetch' in cfg: + self.log.warning( + "Use ExchangeFetchAssignment in config, not ExchangeFetch. Outdated config:\n%s", + '\n'.join( + 'ExchangeFetch.{key} = {value!r}'.format(key=key, value=value) + for key, value in cfg.ExchangeFetch.items() + ) + ) + cfg.ExchangeFetchAssignment.merge(cfg.ExchangeFetch) + del cfg.ExchangeFetch + + super(ExchangeFetchAssignment, self)._load_config(cfg, **kwargs) + + def init_src(self): + if self.coursedir.course_id == '': + self.fail("No course id specified. Re-run with --course flag.") + if not self.authenticator.has_access(self.coursedir.student_id, self.coursedir.course_id): + self.fail("You do not have access to this course.") + + self.course_path = os.path.join(self.root, self.coursedir.course_id) + self.outbound_path = os.path.join(self.course_path, 'outbound') + self.src_path = os.path.join(self.outbound_path, self.coursedir.assignment_id) + if not os.path.isdir(self.src_path): + self._assignment_not_found( + self.src_path, + os.path.join(self.outbound_path, "*")) + if not check_mode(self.src_path, read=True, execute=True): + self.fail("You don't have read permissions for the directory: {}".format(self.src_path)) + + def init_dest(self): + if self.path_includes_course: + root = os.path.join(self.coursedir.course_id, self.coursedir.assignment_id) + else: + root = self.coursedir.assignment_id + self.dest_path = os.path.abspath(os.path.join(self.assignment_dir, root)) + if os.path.isdir(self.dest_path) and not self.replace_missing_files: + self.fail("You already have a copy of the assignment in this directory: {}".format(root)) + + def copy_if_missing(self, src, dest, ignore=None): + filenames = sorted(os.listdir(src)) + if ignore: + bad_filenames = ignore(src, filenames) + filenames = sorted(list(set(filenames) - bad_filenames)) + + for filename in filenames: + srcpath = os.path.join(src, filename) + destpath = os.path.join(dest, filename) + relpath = os.path.relpath(destpath, os.getcwd()) + if not os.path.exists(destpath): + if os.path.isdir(srcpath): + self.log.warning("Creating missing directory '%s'", relpath) + os.mkdir(destpath) + + else: + self.log.warning("Replacing missing file '%s'", relpath) + shutil.copy(srcpath, destpath) + + if os.path.isdir(srcpath): + self.copy_if_missing(srcpath, destpath, ignore=ignore) + + def do_copy(self, src, dest): + """Copy the src dir to the dest dir omitting the self.coursedir.ignore globs.""" + if os.path.isdir(self.dest_path): + self.copy_if_missing(src, dest, ignore=shutil.ignore_patterns(*self.coursedir.ignore)) + else: + shutil.copytree(src, dest, ignore=shutil.ignore_patterns(*self.coursedir.ignore)) + + def copy_files(self): + self.log.info("Source: {}".format(self.src_path)) + self.log.info("Destination: {}".format(self.dest_path)) + self.do_copy(self.src_path, self.dest_path) + self.log.info("Fetched as: {} {}".format(self.coursedir.course_id, self.coursedir.assignment_id)) diff --git a/nbgrader-exchange/nbgrader_k8s_exchange/plugin/fetch_feedback.py b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/fetch_feedback.py new file mode 100644 index 0000000..902c2d5 --- /dev/null +++ b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/fetch_feedback.py @@ -0,0 +1,111 @@ +import os +import shutil +import glob + +from nbgrader.exchange.abc import ExchangeFetchFeedback as ABCExchangeFetchFeedback +from nbgrader.exchange.default import Exchange + +from nbgrader.utils import check_mode, notebook_hash, make_unique_key, get_username + + +class ExchangeFetchFeedback(Exchange, ABCExchangeFetchFeedback): + + def init_src(self): + if self.coursedir.course_id == '': + self.fail("No course id specified. Re-run with --course flag.") + + self.course_path = os.path.join(self.root, self.coursedir.course_id) + + self.cache_path = os.path.join(self.cache, self.coursedir.course_id) + + if self.coursedir.student_id != '*': + # An explicit student id has been specified on the command line; we use it as student_id + if '*' in self.coursedir.student_id or '+' in self.coursedir.student_id: + self.fail("The student ID should contain no '*' nor '+'; got {}".format(self.coursedir.student_id)) + student_id = self.coursedir.student_id + else: + student_id = get_username() + + self.outbound_path = os.path.join(self.course_path, 'feedback_public', student_id) + self.src_path = os.path.join(self.outbound_path) + + if not os.path.isdir(self.src_path): + self._assignment_not_found( + self.src_path, + os.path.join(self.outbound_path, "*")) + if not check_mode(self.src_path, execute=True): + self.fail("You don't have execute permissions for the directory: {}".format(self.src_path)) + + assignment_id = self.coursedir.assignment_id if self.coursedir.assignment_id else '*' + pattern = os.path.join(self.cache_path, '*+{}+*'.format(assignment_id)) + self.log.debug( + "Looking for submissions with pattern: {}".format(pattern)) + + self.feedback_files = [] + submissions = [os.path.split(x)[-1] for x in glob.glob(pattern)] + for submission in submissions: + _, assignment_id, timestamp = submission.split('/')[-1].split('+') + self.log.debug( + "Looking for feedback for '{}/{}' submitted at {}".format( + self.coursedir.course_id, assignment_id, timestamp)) + pattern = os.path.join(self.cache_path, submission, "*.ipynb") + notebooks = glob.glob(pattern) + for notebook in notebooks: + notebook_id = os.path.splitext(os.path.split(notebook)[-1])[0] + unique_key = make_unique_key( + self.coursedir.course_id, + assignment_id, + notebook_id, + student_id, + timestamp) + + # Look for the feedback using new-style of feedback + self.log.debug("Unique key is: {}".format(unique_key)) + nb_hash = notebook_hash(notebook, unique_key) + feedbackpath = os.path.join(self.outbound_path, '{0}.html'.format(nb_hash)) + if os.path.exists(feedbackpath): + self.feedback_files.append((notebook_id, timestamp, feedbackpath)) + self.log.info( + "Found feedback for '{}/{}/{}' submitted at {}".format( + self.coursedir.course_id, assignment_id, notebook_id, timestamp)) + continue + + # If it doesn't exist, try the legacy hashing + nb_hash = notebook_hash(notebook) + feedbackpath = os.path.join(self.outbound_path, '{0}.html'.format(nb_hash)) + if os.path.exists(feedbackpath): + self.feedback_files.append((notebook_id, timestamp, feedbackpath)) + self.log.warning( + "Found legacy feedback for '{}/{}/{}' submitted at {}".format( + self.coursedir.course_id, assignment_id, notebook_id, timestamp)) + continue + + # If we reached here, then there's no feedback available + self.log.warning( + "Could not find feedback for '{}/{}/{}' submitted at {}".format( + self.coursedir.course_id, assignment_id, notebook_id, timestamp)) + + def init_dest(self): + if self.path_includes_course: + root = os.path.join(self.coursedir.course_id, self.coursedir.assignment_id) + else: + root = self.coursedir.assignment_id + self.dest_path = os.path.abspath(os.path.join(self.assignment_dir, root, 'feedback')) + + def do_copy(self, src, dest): + for notebook_id, timestamp, feedbackpath in self.feedback_files: + dest_with_timestamp = os.path.join(dest, timestamp) + if not os.path.isdir(dest_with_timestamp): + os.makedirs(dest_with_timestamp) + new_name = notebook_id + '.html' + html_file = os.path.join(dest_with_timestamp, new_name) + self.log.debug("Copying feedback from {} to {}".format(feedbackpath, html_file)) + if os.path.exists(html_file): + self.log.debug("Overwriting existing feedback: {}".format(html_file)) + shutil.copy(feedbackpath, html_file) + self.log.info("Fetched feedback: {}".format(html_file)) + + def copy_files(self): + self.log.info("Source: {}".format(self.src_path)) + self.log.info("Destination: {}".format(self.dest_path)) + self.do_copy(self.src_path, self.dest_path) diff --git a/nbgrader-exchange/nbgrader_k8s_exchange/plugin/list.py b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/list.py new file mode 100644 index 0000000..f435e25 --- /dev/null +++ b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/list.py @@ -0,0 +1,241 @@ +import os +import glob +import shutil +import re +import hashlib + +from nbgrader.exchange.abc import ExchangeList as ABCExchangeList +from nbgrader.utils import notebook_hash, make_unique_key +from .exchange import Exchange + +def _checksum(path): + m = hashlib.md5() + m.update(open(path, 'rb').read()) + return m.hexdigest() + + +class ExchangeList(ABCExchangeList, Exchange): + + def init_src(self): + pass + + def init_dest(self): + course_id = self.coursedir.course_id if self.coursedir.course_id else '*' + assignment_id = self.coursedir.assignment_id if self.coursedir.assignment_id else '*' + student_id = self.coursedir.student_id if self.coursedir.student_id else '*' + + if self.inbound: + pattern = os.path.join(self.root, course_id, 'inbound', student_id, '{}+{}+*'.format(student_id, assignment_id)) + elif self.cached: + pattern = os.path.join(self.cache, course_id, '{}+{}+*'.format(student_id, assignment_id)) + else: + pattern = os.path.join(self.root, course_id, 'outbound', '{}'.format(assignment_id)) + + self.assignments = sorted(glob.glob(pattern)) + + def parse_assignment(self, assignment): + if self.inbound: + regexp = r".*/(?P.*)/inbound/(?P.*)/(?P[^+]*)\+(?P[^+]*)\+(?P[^+]*)(?P\+.*)?" + elif self.cached: + regexp = r".*/(?P.*)/(?P.*)\+(?P.*)\+(?P.*)" + else: + regexp = r".*/(?P.*)/outbound/(?P.*)" + + m = re.match(regexp, assignment) + if m is None: + raise RuntimeError("Could not match '%s' with regexp '%s'", assignment, regexp) + return m.groupdict() + + def format_inbound_assignment(self, info): + msg = "{course_id} {student_id} {assignment_id} {timestamp}".format(**info) + if info['status'] == 'submitted': + if info['has_local_feedback'] and not info['feedback_updated']: + msg += " (feedback already fetched)" + elif info['has_exchange_feedback']: + msg += " (feedback ready to be fetched)" + else: + msg += " (no feedback available)" + return msg + + def format_outbound_assignment(self, info): + msg = "{course_id} {assignment_id}".format(**info) + if os.path.exists(info['assignment_id']): + msg += " (already downloaded)" + return msg + + def copy_files(self): + pass + + def parse_assignments(self): + # None means that the student has access to any course that might exist + courses = None + + assignments = [] + for path in self.assignments: + info = self.parse_assignment(path) + if courses is not None and info['course_id'] not in courses: + continue + + if self.path_includes_course: + assignment_dir = os.path.join(self.assignment_dir, info['course_id'], info['assignment_id']) + else: + assignment_dir = os.path.join(self.assignment_dir, info['assignment_id']) + + if self.inbound or self.cached: + info['status'] = 'submitted' + info['path'] = path + elif os.path.exists(assignment_dir): + info['status'] = 'fetched' + info['path'] = os.path.abspath(assignment_dir) + else: + info['status'] = 'released' + info['path'] = path + + if self.remove: + info['status'] = 'removed' + + notebooks = sorted(glob.glob(os.path.join(info['path'], '*.ipynb'))) + if not notebooks: + self.log.warning("No notebooks found in {}".format(info['path'])) + + info['notebooks'] = [] + for notebook in notebooks: + nb_info = { + 'notebook_id': os.path.splitext(os.path.split(notebook)[1])[0], + 'path': os.path.abspath(notebook) + } + if info['status'] != 'submitted': + info['notebooks'].append(nb_info) + continue + + nb_info['has_local_feedback'] = False + nb_info['has_exchange_feedback'] = False + nb_info['local_feedback_path'] = None + nb_info['feedback_updated'] = False + + # Check whether feedback has been fetched already. + local_feedback_dir = os.path.join( + assignment_dir, 'feedback', info['timestamp']) + local_feedback_path = os.path.join( + local_feedback_dir, '{0}.html'.format(nb_info['notebook_id'])) + has_local_feedback = os.path.isfile(local_feedback_path) + if has_local_feedback: + local_feedback_checksum = _checksum(local_feedback_path) + else: + local_feedback_checksum = None + + # Also look to see if there is feedback available to fetch. + unique_key = make_unique_key( + info['course_id'], + info['assignment_id'], + nb_info['notebook_id'], + info['student_id'], + info['timestamp']) + self.log.debug("Unique key is: {}".format(unique_key)) + nb_hash = notebook_hash(notebook, unique_key) + exchange_feedback_path = os.path.join( + self.root, info['course_id'], 'feedback', '{0}.html'.format(nb_hash)) + has_exchange_feedback = os.path.isfile(exchange_feedback_path) + if not has_exchange_feedback: + # Try looking for legacy feedback. + nb_hash = notebook_hash(notebook) + exchange_feedback_path = os.path.join( + self.root, info['course_id'], 'feedback', '{0}.html'.format(nb_hash)) + has_exchange_feedback = os.path.isfile(exchange_feedback_path) + if has_exchange_feedback: + exchange_feedback_checksum = _checksum(exchange_feedback_path) + else: + exchange_feedback_checksum = None + + nb_info['has_local_feedback'] = has_local_feedback + nb_info['has_exchange_feedback'] = has_exchange_feedback + if has_local_feedback: + nb_info['local_feedback_path'] = local_feedback_path + if has_local_feedback and has_exchange_feedback: + nb_info['feedback_updated'] = exchange_feedback_checksum != local_feedback_checksum + info['notebooks'].append(nb_info) + + if info['status'] == 'submitted': + if info['notebooks']: + has_local_feedback = all([nb['has_local_feedback'] for nb in info['notebooks']]) + has_exchange_feedback = all([nb['has_exchange_feedback'] for nb in info['notebooks']]) + feedback_updated = any([nb['feedback_updated'] for nb in info['notebooks']]) + else: + has_local_feedback = False + has_exchange_feedback = False + feedback_updated = False + + info['has_local_feedback'] = has_local_feedback + info['has_exchange_feedback'] = has_exchange_feedback + info['feedback_updated'] = feedback_updated + if has_local_feedback: + full_path_assignment_dir = os.path.abspath(assignment_dir) + if os.path.exists(full_path_assignment_dir): + info['local_feedback_path'] = os.path.join( + full_path_assignment_dir, 'feedback', info['timestamp']) + else: + info['local_feedback_path'] = os.path.join( + assignment_dir, 'feedback', info['timestamp']) + else: + info['local_feedback_path'] = None + + assignments.append(info) + + # partition the assignments into groups for course/student/assignment + if self.inbound or self.cached: + _get_key = lambda info: (info['course_id'], info['student_id'], info['assignment_id']) + _match_key = lambda info, key: ( + info['course_id'] == key[0] and + info['student_id'] == key[1] and + info['assignment_id'] == key[2]) + assignment_keys = sorted(list(set([_get_key(info) for info in assignments]))) + assignment_submissions = [] + for key in assignment_keys: + submissions = [x for x in assignments if _match_key(x, key)] + submissions = sorted(submissions, key=lambda x: x['timestamp']) + info = { + 'course_id': key[0], + 'student_id': key[1], + 'assignment_id': key[2], + 'status': submissions[0]['status'], + 'submissions': submissions + } + assignment_submissions.append(info) + assignments = assignment_submissions + + return assignments + + def list_files(self): + """List files.""" + assignments = self.parse_assignments() + + if self.inbound or self.cached: + self.log.info("Submitted assignments:") + for assignment in assignments: + for info in assignment['submissions']: + self.log.info(self.format_inbound_assignment(info)) + else: + self.log.info("Released assignments:") + for info in assignments: + self.log.info(self.format_outbound_assignment(info)) + + return assignments + + def remove_files(self): + """List and remove files.""" + assignments = self.parse_assignments() + + if self.inbound or self.cached: + self.log.info("Removing submitted assignments:") + for assignment in assignments: + for info in assignment['submissions']: + self.log.info(self.format_inbound_assignment(info)) + else: + self.log.info("Removing released assignments:") + for info in assignments: + self.log.info(self.format_outbound_assignment(info)) + + for assignment in self.assignments: + shutil.rmtree(assignment) + + return assignments diff --git a/nbgrader-exchange/nbgrader_k8s_exchange/plugin/release_assignment.py b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/release_assignment.py new file mode 100644 index 0000000..efc7d53 --- /dev/null +++ b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/release_assignment.py @@ -0,0 +1,113 @@ +import os +import shutil +from stat import ( + S_IRUSR, S_IWUSR, S_IXUSR, + S_IRGRP, S_IWGRP, S_IXGRP, + S_IROTH, S_IWOTH, S_IXOTH, + S_ISGID, ST_MODE +) + + +from nbgrader.exchange.abc import ExchangeReleaseAssignment as ABCExchangeReleaseAssignment +from nbgrader.exchange.default import Exchange + + +class ExchangeReleaseAssignment(Exchange, ABCExchangeReleaseAssignment): + + def _load_config(self, cfg, **kwargs): + if 'ExchangeRelease' in cfg: + self.log.warning( + "Use ExchangeReleaseAssignment in config, not ExchangeRelease. Outdated config:\n%s", + '\n'.join( + 'ExchangeRelease.{key} = {value!r}'.format(key=key, value=value) + for key, value in cfg.ExchangeRelease.items() + ) + ) + cfg.ExchangeReleaseAssignment.merge(cfg.ExchangeRelease) + del cfg.ExchangeRelease + + super(ExchangeReleaseAssignment, self)._load_config(cfg, **kwargs) + + def ensure_root(self): + perms = S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IWOTH|S_IXOTH|((S_IWGRP|S_ISGID) if self.coursedir.groupshared else 0) + + # if root doesn't exist, create it and set permissions + if not os.path.exists(self.root): + self.log.warning("Creating exchange directory: {}".format(self.root)) + try: + os.makedirs(self.root) + os.chmod(self.root, perms) + except PermissionError: + self.fail("Could not create {}, permission denied.".format(self.root)) + + else: + old_perms = oct(os.stat(self.root)[ST_MODE] & 0o777) + new_perms = oct(perms & 0o777) + if old_perms != new_perms: + self.log.warning( + "Permissions for exchange directory ({}) are invalid, changing them from {} to {}".format( + self.root, old_perms, new_perms)) + try: + os.chmod(self.root, perms) + except PermissionError: + self.fail("Could not change permissions of {}, permission denied.".format(self.root)) + + def init_src(self): + self.src_path = self.coursedir.format_path(self.coursedir.release_directory, '.', self.coursedir.assignment_id) + if not os.path.isdir(self.src_path): + source = self.coursedir.format_path(self.coursedir.source_directory, '.', self.coursedir.assignment_id) + if os.path.isdir(source): + # Looks like the instructor forgot to assign + self.fail("Assignment found in '{}' but not '{}', run `nbgrader generate_assignment` first.".format( + source, self.src_path)) + else: + self._assignment_not_found( + self.src_path, + self.coursedir.format_path(self.coursedir.release_directory, '.', '*')) + + def init_dest(self): + if self.coursedir.course_id == '': + self.fail("No course id specified. Re-run with --course flag.") + + self.course_path = os.path.join(self.root, self.coursedir.course_id) + self.outbound_path = os.path.join(self.course_path, 'outbound') + self.inbound_path = os.path.join(self.course_path, 'inbound') + self.dest_path = os.path.join(self.outbound_path, self.coursedir.assignment_id) + # 0755 + # groupshared: +2040 + self.ensure_directory( + self.course_path, + S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH|((S_ISGID|S_IWGRP) if self.coursedir.groupshared else 0) + ) + # 0755 + # groupshared: +2040 + self.ensure_directory( + self.outbound_path, + S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH|((S_ISGID|S_IWGRP) if self.coursedir.groupshared else 0) + ) + # 0733 with set GID so student submission will have the instructors group + # groupshared: +0040 + self.ensure_directory( + self.inbound_path, + S_ISGID|S_IRUSR|S_IWUSR|S_IXUSR|S_IWGRP|S_IXGRP|S_IWOTH|S_IXOTH|(S_IRGRP if self.coursedir.groupshared else 0) + ) + + def copy_files(self): + if os.path.isdir(self.dest_path): + if self.force: + self.log.info("Overwriting files: {} {}".format( + self.coursedir.course_id, self.coursedir.assignment_id + )) + shutil.rmtree(self.dest_path) + else: + self.fail("Destination already exists, add --force to overwrite: {} {}".format( + self.coursedir.course_id, self.coursedir.assignment_id + )) + self.log.info("Source: {}".format(self.src_path)) + self.log.info("Destination: {}".format(self.dest_path)) + self.do_copy(self.src_path, self.dest_path) + self.set_perms( + self.dest_path, + fileperms=(S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH|(S_IWGRP if self.coursedir.groupshared else 0)), + dirperms=(S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH|((S_ISGID|S_IWGRP) if self.coursedir.groupshared else 0))) + self.log.info("Released as: {} {}".format(self.coursedir.course_id, self.coursedir.assignment_id)) diff --git a/nbgrader-exchange/nbgrader_k8s_exchange/plugin/release_feedback.py b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/release_feedback.py new file mode 100644 index 0000000..5d6ce9b --- /dev/null +++ b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/release_feedback.py @@ -0,0 +1,92 @@ +import os +import shutil +import glob +import re +from stat import S_IRUSR, S_IWUSR, S_IXUSR, S_IRGRP, S_IWGRP, S_IXGRP, S_IXOTH, S_ISGID + +from nbgrader.exchange.abc import ExchangeReleaseFeedback as ABCExchangeReleaseFeedback +from .exchange import Exchange +from nbgrader.utils import notebook_hash, make_unique_key + + +class ExchangeReleaseFeedback(Exchange, ABCExchangeReleaseFeedback): + + def init_src(self): + student_id = self.coursedir.student_id if self.coursedir.student_id else '*' + self.src_path = self.coursedir.format_path( + self.coursedir.feedback_directory, student_id, + self.coursedir.assignment_id) + + def init_dest(self): + if self.coursedir.course_id == '': + self.fail("No course id specified. Re-run with --course flag.") + + self.course_path = os.path.join(self.root, self.coursedir.course_id) + self.outbound_feedback_path = os.path.join(self.course_path, "feedback_public") + self.dest_path = os.path.join(self.outbound_feedback_path) + # 0755 + self.ensure_directory( + self.outbound_feedback_path, + (S_IRUSR | S_IWUSR | S_IXUSR | S_IXGRP | S_IXOTH | + ((S_IRGRP|S_IWGRP|S_ISGID) if self.coursedir.groupshared else 0)) + ) + + def copy_files(self): + if self.coursedir.student_id_exclude: + exclude_students = set(self.coursedir.student_id_exclude.split(',')) + else: + exclude_students = set() + + html_files = glob.glob(os.path.join(self.src_path, "*.html")) + for html_file in html_files: + regexp = re.escape(os.path.sep).join([ + self.coursedir.format_path( + self.coursedir.feedback_directory, + "(?P.*)", + self.coursedir.assignment_id, escape=True), + "(?P.*).html" + ]) + + m = re.match(regexp, html_file) + if m is None: + msg = "Could not match '%s' with regexp '%s'" % (html_file, regexp) + self.log.error(msg) + continue + + gd = m.groupdict() + student_id = gd['student_id'] + notebook_id = gd['notebook_id'] + if student_id in exclude_students: + self.log.debug("Skipping student '{}'".format(student_id)) + continue + + feedback_dir = os.path.split(html_file)[0] + submission_dir = self.coursedir.format_path( + self.coursedir.submitted_directory, student_id, + self.coursedir.assignment_id) + + timestamp = open(os.path.join(feedback_dir, 'timestamp.txt')).read() + nbfile = os.path.join(submission_dir, "{}.ipynb".format(notebook_id)) + unique_key = make_unique_key( + self.coursedir.course_id, + self.coursedir.assignment_id, + notebook_id, + student_id, + timestamp) + + self.log.debug("Unique key is: {}".format(unique_key)) + checksum = notebook_hash(nbfile, unique_key) + dest_dir = os.path.join(self.dest_path, student_id) + + self.ensure_directory( + dest_dir, + (S_IRUSR | S_IWUSR | S_IXUSR | S_IXGRP | S_IXOTH | + ((S_IRGRP|S_IWGRP|S_ISGID) if self.coursedir.groupshared else 0)) + ) + + dest = os.path.join(dest_dir, "{}.html".format(checksum)) + + self.log.info("Releasing feedback for student '{}' on assignment '{}/{}/{}' ({})".format( + student_id, self.coursedir.course_id, self.coursedir.assignment_id, notebook_id, timestamp)) + shutil.copy(html_file, dest) + self.log.info("Feedback released to: {}".format(dest)) diff --git a/nbgrader-exchange/nbgrader_k8s_exchange/plugin/submit.py b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/submit.py new file mode 100644 index 0000000..c05b3f7 --- /dev/null +++ b/nbgrader-exchange/nbgrader_k8s_exchange/plugin/submit.py @@ -0,0 +1,160 @@ +import base64 +import os +from stat import ( + S_IRUSR, S_IWUSR, S_IXUSR, + S_IRGRP, S_IWGRP, S_IXGRP, + S_IROTH, S_IWOTH, S_IXOTH, S_ISGID +) +from textwrap import dedent + +from nbgrader.exchange.abc import ExchangeSubmit as ABCExchangeSubmit +from traitlets import Bool + +from .exchange import Exchange +from nbgrader.utils import get_username, check_mode, find_all_notebooks + + +class ExchangeSubmit(Exchange, ABCExchangeSubmit): + + add_random_string = Bool( + True, + help=dedent( + "Whether to add a random string on the end of the submission." + ) + ).tag(config=True) + + def init_src(self): + if self.path_includes_course: + root = os.path.join(self.coursedir.course_id, self.coursedir.assignment_id) + other_path = os.path.join(self.coursedir.course_id, "*") + else: + root = self.coursedir.assignment_id + other_path = "*" + self.src_path = os.path.abspath(os.path.join(self.assignment_dir, root)) + self.coursedir.assignment_id = os.path.split(self.src_path)[-1] + if not os.path.isdir(self.src_path): + self._assignment_not_found(self.src_path, os.path.abspath(other_path)) + + def init_dest(self): + if self.coursedir.course_id == '': + self.fail("No course id specified. Re-run with --course flag.") + if not self.authenticator.has_access(self.coursedir.student_id, self.coursedir.course_id): + self.fail("You do not have access to this course.") + + self.cache_path = os.path.join(self.cache, self.coursedir.course_id) + if self.coursedir.student_id != '*': + # An explicit student id has been specified on the command line; we use it as student_id + if '*' in self.coursedir.student_id or '+' in self.coursedir.student_id: + self.fail("The student ID should contain no '*' nor '+'; got {}".format(self.coursedir.student_id)) + student_id = self.coursedir.student_id + else: + student_id = get_username() + + self.inbound_path = os.path.join(self.root, self.coursedir.course_id, 'inbound', student_id) + self.ensure_directory( + self.inbound_path, + S_ISGID|S_IRUSR|S_IWUSR|S_IXUSR|S_IWGRP|S_IXGRP|S_IWOTH|S_IXOTH|(S_IRGRP if self.coursedir.groupshared else 0) + ) + + if self.add_random_string: + random_str = base64.urlsafe_b64encode(os.urandom(9)).decode('ascii') + self.assignment_filename = '{}+{}+{}+{}'.format( + student_id, self.coursedir.assignment_id, self.timestamp, random_str) + else: + self.assignment_filename = '{}+{}+{}'.format( + student_id, self.coursedir.assignment_id, self.timestamp) + + def init_release(self): + if self.coursedir.course_id == '': + self.fail("No course id specified. Re-run with --course flag.") + + course_path = os.path.join(self.root, self.coursedir.course_id) + outbound_path = os.path.join(course_path, 'outbound') + self.release_path = os.path.join(outbound_path, self.coursedir.assignment_id) + if not os.path.isdir(self.release_path): + self.fail("Assignment not found: {}".format(self.release_path)) + if not check_mode(self.release_path, read=True, execute=True): + self.fail("You don't have read permissions for the directory: {}".format(self.release_path)) + + def check_filename_diff(self): + released_notebooks = find_all_notebooks(self.release_path) + submitted_notebooks = find_all_notebooks(self.src_path) + + # Look for missing notebooks in submitted notebooks + missing = False + release_diff = list() + for filename in released_notebooks: + if filename in submitted_notebooks: + release_diff.append("{}: {}".format(filename, 'FOUND')) + else: + missing = True + release_diff.append("{}: {}".format(filename, 'MISSING')) + + # Look for extra notebooks in submitted notebooks + extra = False + submitted_diff = list() + for filename in submitted_notebooks: + if filename in released_notebooks: + submitted_diff.append("{}: {}".format(filename, 'OK')) + else: + extra = True + submitted_diff.append("{}: {}".format(filename, 'EXTRA')) + + if missing or extra: + diff_msg = ( + "Expected:\n\t{}\nSubmitted:\n\t{}".format( + '\n\t'.join(release_diff), + '\n\t'.join(submitted_diff), + ) + ) + if missing and self.strict: + self.fail( + "Assignment {} not submitted. " + "There are missing notebooks for the submission:\n{}" + "".format(self.coursedir.assignment_id, diff_msg) + ) + else: + self.log.warning( + "Possible missing notebooks and/or extra notebooks " + "submitted for assignment {}:\n{}" + "".format(self.coursedir.assignment_id, diff_msg) + ) + + def copy_files(self): + self.init_release() + + dest_path = os.path.join(self.inbound_path, self.assignment_filename) + if self.add_random_string: + cache_path = os.path.join(self.cache_path, self.assignment_filename.rsplit('+', 1)[0]) + else: + cache_path = os.path.join(self.cache_path, self.assignment_filename) + + self.log.info("Source: {}".format(self.src_path)) + self.log.info("Destination: {}".format(dest_path)) + + # copy to the real location + self.check_filename_diff() + self.do_copy(self.src_path, dest_path) + with open(os.path.join(dest_path, "timestamp.txt"), "w") as fh: + fh.write(self.timestamp) + self.set_perms( + dest_path, + fileperms=(S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH), + dirperms=(S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)) + + # Make this 0777=ugo=rwx so the instructor can delete later. Hidden from other users by the timestamp. + os.chmod( + dest_path, + S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IWGRP|S_IXGRP|S_IROTH|S_IWOTH|S_IXOTH + ) + + # also copy to the cache + if not os.path.isdir(self.cache_path): + os.makedirs(self.cache_path) + self.do_copy(self.src_path, cache_path) + with open(os.path.join(cache_path, "timestamp.txt"), "w") as fh: + fh.write(self.timestamp) + + self.log.info("Submitted as: {} {} {}".format( + self.coursedir.course_id, self.coursedir.assignment_id, str(self.timestamp) + )) diff --git a/nbgrader-exchange/pyproject.toml b/nbgrader-exchange/pyproject.toml new file mode 100644 index 0000000..504c165 --- /dev/null +++ b/nbgrader-exchange/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "nbgrader-k8s-exchange" +version = "0.0.1" +authors = [ + { name="CERIT-SC", email="support@cerit-sc.cz"}, +] +description = "K8s compatibe exchange for nbgrader" +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://github.com/CERIT-SC/nbgrader-k8s" +"Bug Tracker" = "https://github.com/CERIT-SC/nbgrader-k8s/issues" + +dependencies = [ + "nbgrader==0.9.1", +] \ No newline at end of file