run_labeling_job.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import os\n",
    "from collections import namedtuple\n",
    "from collections import defaultdict\n",
    "from collections import Counter\n",
    "import itertools\n",
    "import json\n",
    "import random\n",
    "import time\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.backends.backend_pdf import PdfPages\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import boto3\n",
    "import sagemaker\n",
    "from urllib.parse import urlparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all task-specific variables are set in this cell.\n",
    "\n",
    "# info about the data to be annotated and the annotation interface.\n",
    "BUCKET = 'white-100' #s3 bucket containing the data to be annotated.\n",
    "MANIFEST = 'white_mouse_100' #name of the data manifest inside that bucket.\n",
    "UITEMPLATE = 's3://white-mouse-test/final_keypoint_unique_label_white.template' #template file for the annotation interface.\n",
    "\n",
    "# additional task info/instructions\n",
    "task_description = 'Please label each body part of interest'\n",
    "task_keywords = ['pose', 'keypoints','animal']\n",
    "task_title = task_description\n",
    "job_name = '{}-'.format(BUCKET) + str(int(time.time()))\n",
    "\n",
    "# how much each annotator is compensated per frame. Check AWS documentation for permissable values.\n",
    "price_dollars = 0\n",
    "price_cents = 6\n",
    "price_tenthcent = 0\n",
    "\n",
    "# Specify ARNs for resources needed to run an image classification job.\n",
    "prehuman_arn = 'arn:aws:lambda:us-east-2:522254924758:function:serverlessrepo-aws-sagema-GtRecipePreHumanTaskFunc-1L5Z2249KDMHU'\n",
    "acs_arn      = 'arn:aws:lambda:us-east-2:522254924758:function:serverlessrepo-aws-sagema-GtRecipeAnnotationConsol-1JQ1AEBC0TNB3'\n",
    "workteam_arn = 'arn:aws:sagemaker:us-east-2:394669845002:workteam/public-crowd/default'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make sure the bucket is in the same region as this notebook.\n",
    "role = sagemaker.get_execution_role()\n",
    "region = boto3.session.Session().region_name\n",
    "s3 = boto3.client('s3')\n",
    "bucket_region = s3.head_bucket(Bucket=BUCKET)['ResponseMetadata']['HTTPHeaders']['x-amz-bucket-region']\n",
    "assert bucket_region == region, \"You S3 bucket {} and this notebook need to be in the same region.\".format(BUCKET)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, package task info and submit!\n",
    "\n",
    "# Info about the workforce and task duration.\n",
    "human_task_config = {\n",
    "      \"AnnotationConsolidationConfig\": {\n",
    "        \"AnnotationConsolidationLambdaArn\": acs_arn,\n",
    "      },\n",
    "      \"PreHumanTaskLambdaArn\": prehuman_arn,\n",
    "      \"MaxConcurrentTaskCount\": 200, # 200 images will be sent at a time to the workteam.\n",
    "      \"NumberOfHumanWorkersPerDataObject\": 6, # 3 separate workers will be required to label each image.\n",
    "      \"TaskAvailabilityLifetimeInSeconds\": 12*60*60, # seconds to complete all pending tasks.\n",
    "      \"TaskDescription\": task_description,\n",
    "      \"TaskKeywords\": task_keywords,\n",
    "      \"TaskTimeLimitInSeconds\": 300, # Each image must be labeled within 5 minutes.\n",
    "      \"TaskTitle\": task_title,\n",
    "      \"UiConfig\": {\n",
    "        \"UiTemplateS3Uri\": UITEMPLATE,\n",
    "      }\n",
    "    }\n",
    "\n",
    "\n",
    "# Specifies the workforce and compensation.\n",
    "human_task_config[\"PublicWorkforceTaskPrice\"] = {\n",
    "    \"AmountInUsd\": {\n",
    "       \"Dollars\": price_dollars,\n",
    "       \"Cents\": price_cents,\n",
    "       \"TenthFractionsOfACent\": price_tenthcent,\n",
    "    }\n",
    "} \n",
    "human_task_config[\"WorkteamArn\"] = workteam_arn\n",
    "\n",
    "# Info about the task dataset.\n",
    "ground_truth_request = {\n",
    "        \"InputConfig\" : {\n",
    "          \"DataSource\": {\n",
    "            \"S3DataSource\": {\n",
    "              \"ManifestS3Uri\": 's3://{}/{}.manifest'.format(BUCKET,MANIFEST),\n",
    "            }\n",
    "          },\n",
    "          \"DataAttributes\": {\n",
    "            \"ContentClassifiers\": [\n",
    "              \"FreeOfPersonallyIdentifiableInformation\",\n",
    "              \"FreeOfAdultContent\"\n",
    "            ]\n",
    "          },  \n",
    "        },\n",
    "        \"OutputConfig\" : {\n",
    "          \"S3OutputPath\": 's3://{}-output/'.format(BUCKET),\n",
    "        },\n",
    "        \"HumanTaskConfig\" : human_task_config,\n",
    "        \"LabelingJobName\": job_name,\n",
    "        \"RoleArn\": role, \n",
    "        \"LabelAttributeName\": \"annotatedResult\",\n",
    "    }\n",
    "\n",
    "# Submit the job.\n",
    "sagemaker_client = boto3.client('sagemaker')\n",
    "sagemaker_client.create_labeling_job(**ground_truth_request)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}