google cloud/google cloud with python/cloud.google.compute.instances.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# gcloud.google.compute.instances\n",
    "This notebook create, delete start and stop instances in google compute cloud service"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#imports\n",
    "import googleapiclient.discovery\n",
    "from apiclient.discovery import build\n",
    "import ipywidgets as widgets\n",
    "import re\n",
    "import json\n",
    "#we use pandas.DataFrame for printing arrays and dictionnary nicely\n",
    "from pandas import DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load credentials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %load getCredentialsFromFile.py\n",
    "\n",
    "\n",
    "def getCredentials():\n",
    "    from  oauth2client import file\n",
    "    import httplib2\n",
    "    import ipywidgets as widgets\n",
    "    print(\"Getting the credentials from file...\")\n",
    "    storage = file.Storage(\"oauth2.dat\")\n",
    "    credentials=storage.get()\n",
    "    if credentials is None or credentials.invalid:\n",
    "        print( '❗')\n",
    "        display(widgets.Valid(\n",
    "            value=False,\n",
    "            description='Credentials are ',\n",
    "            disabled=False))\n",
    "        display(widgets.HTML('go create a credential valid file here: <a target=\"_blank\" href=\"cloud.google.auth.ipynb.ipynb\">gcloud authorization notebook</a> and try again'))\n",
    "    else:\n",
    "        http_auth = credentials.authorize(httplib2.Http())\n",
    "        print('✅ Ok')\n",
    "        return credentials\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "credentials=getCredentials()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create services"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "compute_service = build('compute', 'v1', credentials=credentials)\n",
    "resource_service = build('cloudresourcemanager', 'v1', credentials=credentials)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose projectId and zone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %load chooseProjectId.py\n",
    "#projectId is the variable that will contains the projectId that will be used in the API calls\n",
    "projectId=None\n",
    "\n",
    "#list the existing projects \n",
    "projects=resource_service.projects().list().execute()\n",
    "#we create a dictionaray name:projectId foe a dropdown list widget\n",
    "projectsList={project['name']:project['projectId'] for project in projects['projects']}\n",
    "projectsList['None']='invalid'\n",
    "\n",
    "#the dropdownlist widget\n",
    "projectWidget=widgets.Dropdown(options=projectsList,description='Choose your Project',value='invalid')\n",
    "#a valid widget that get valid when a project is selected\n",
    "projectIdValid=widgets.Valid(value=False,description='')\n",
    "display(widgets.Box([projectWidget,projectIdValid]))\n",
    "\n",
    "def projectValueChange(sender):\n",
    "    if projectWidget.value!='invalid':\n",
    "        #when a valid project is selected ,the gloabl variable projectId is set \n",
    "        projectIdValid.value=True\n",
    "        projectIdValid.description=projectWidget.value\n",
    "        global projectId\n",
    "        projectId=projectWidget.value    \n",
    "    else:\n",
    "        projectIdValid.value=False\n",
    "        projectIdValid.description=''\n",
    "projectWidget.observe(projectValueChange, 'value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# %load chooseZone.py\n",
    "#zone is the variable that will contains the zone that will be used in the API calls\n",
    "zone=None\n",
    "\n",
    "#list the existing zones \n",
    "zones=compute_service.zones().list(project=projectId).execute()\n",
    "#list that will contains the zones for a dropdown list\n",
    "zonesList=[item['name'] for item in zones['items']]\n",
    "zonesList.append('none')\n",
    "\n",
    "#the dropdownlist widget\n",
    "zoneWidget=widgets.Dropdown(options=zonesList,value='none',description='Choose your Zone:')\n",
    "zoneValid=widgets.Valid(value=False,description='')\n",
    "display(widgets.Box([zoneWidget,zoneValid]))\n",
    "\n",
    "def zoneValueChange(sender):\n",
    "    if zoneWidget.value!='none':\n",
    "        #when a vail zone is slected, the variable zone is set\n",
    "        zoneValid.value=True\n",
    "        zoneValid.description=zoneWidget.value\n",
    "        global zone\n",
    "        zone=zoneWidget.value   \n",
    "    else:\n",
    "        zoneValid.value=False\n",
    "        zoneValid.description=''\n",
    "        \n",
    "zoneWidget.observe(zoneValueChange, 'value')    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a new instance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### - choosing the disk image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "image_response = compute_service.images().getFromFamily(\n",
    "        project='debian-cloud', family='debian-8').execute()\n",
    "source_disk_image = image_response['selfLink']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### - choosing the machineType"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "machineType=None\n",
    "\n",
    "machineTypes=compute_service.machineTypes().list(project=projectId,zone=zone).execute()\n",
    "\n",
    "machineTypesList=[item['name'] for item in machineTypes['items']]\n",
    "machineTypesList.append('none')\n",
    "\n",
    "machineTypesWidget=widgets.Dropdown(options=machineTypesList,value='none',description='Choose your MachineType:')\n",
    "machineTypesValid=widgets.Valid(value=False,description='')\n",
    "display(widgets.Box([machineTypesWidget,machineTypesValid]))\n",
    "\n",
    "def machineTypeValueChange(sender):\n",
    "    if machineTypesWidget.value!='none':\n",
    "        machineTypesValid.value=True\n",
    "        machineTypesValid.description=machineTypesWidget.value\n",
    "        global machineType\n",
    "        machineType=machineTypesWidget.value  \n",
    "    else:\n",
    "        machineTypesValid.value=True\n",
    "        machineTypesValid.description=''\n",
    "\n",
    "machineTypesWidget.observe(machineTypeValueChange, 'value') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### - choose an instance name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "instanceName=None\n",
    "\n",
    "# instanceName have to validates this regexp\n",
    "instanceNameControl=re.compile(r'^(?:[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?)$')\n",
    "\n",
    "#the widgets\n",
    "instanceNameWidget=widgets.Text(description=\"Name for the new instance:\")\n",
    "valid=widgets.Valid(value=False,description='',disabled=False)\n",
    "display(widgets.Box([instanceNameWidget,valid]))\n",
    "\n",
    "def instanceNameValueChange(sender):\n",
    "    if instanceNameWidget.value!=\"\":\n",
    "        if instanceNameControl.match(instanceNameWidget.value):\n",
    "            #when the entered text valid the regexp we set the \n",
    "            valid.value=True\n",
    "            valid.description='OK'\n",
    "            global instanceName\n",
    "            instanceName=instanceNameWidget.value\n",
    "        else:\n",
    "            valid.value=False\n",
    "            valid.description=\"The instance name has to verify the regexp '(?:[a-z](?:[-a-z0-9]{0,61}[a-z0-9])?)'\"\n",
    "    else:\n",
    "        valid.value=False\n",
    "        valid.description=''\n",
    "\n",
    "\n",
    "instanceNameWidget.observe(instanceNameValueChange, 'value')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### - creating the config for the new instance\n",
    "\n",
    "The name, machineType and disk are set accordingly to the previous steps.\n",
    "\n",
    "With scheduling.preemptible to true we choose a preemptible instance (cheaper ;-) )\n",
    "\n",
    "You can adjust labels to your needs.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "config= {'name':instanceName,\n",
    "        'machineType': \"zones/%(zone)s/machineTypes/%(machineType)s\" %{'zone':zone,'machineType':machineType},\n",
    "         'disks':[ \n",
    "            \n",
    "             {\n",
    "                 'boot':True,\n",
    "                 'autoDelete':True,\n",
    "                 'initializeParams':{\n",
    "                     'sourceImage':source_disk_image\n",
    "                 }\n",
    "             }],\n",
    "         'scheduling':\n",
    "            {\n",
    "             'preemptible': True\n",
    "            },\n",
    "         'networkInterfaces':[{\n",
    "             'network':'global/networks/default',\n",
    "             'accessConfigs': [\n",
    "                    {'type':'ONE_TO_ONE_NAT','name':'ExternalNAT'}\n",
    "                     ]\n",
    "             }],\n",
    "         'serviceAccounts':[{\n",
    "             'email':'default',\n",
    "             'scopes':[\n",
    "                'https://www.googleapis.com/auth/devstorage.read_write',\n",
    "                'https://www.googleapis.com/auth/logging.write'\n",
    "            ]\n",
    "         }],\n",
    "         \"labels\": {\n",
    "            \"env\": \"test\",\n",
    "            \"created-by\": \"jupyter-notebooks-cloud-google-compute-instances\"\n",
    "          },         \n",
    "        }\n",
    "\n",
    "#print(json.dumps(config, indent=2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### - executing the API call"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#a progress widget will present the progress of the operation\n",
    "progress=widgets.IntProgress(value=0,min=0,max=3,step=1,description=':',bar_style='warning')\n",
    "display(progress)\n",
    "\n",
    "#executing the insert operation\n",
    "operation = compute_service.instances().insert(project=projectId,\n",
    "                                  zone=zone,\n",
    "                                  body=config\n",
    "                                  ).execute()\n",
    "\n",
    "def updateProgress(result,progress=progress):\n",
    "    #updating the progress widget with the result of the operation\n",
    "    if result['status']== 'PENDING':\n",
    "        progress.value=1\n",
    "        progress.bar_style='warning'\n",
    "        progress.description=result['status']\n",
    "    elif result['status']== 'RUNNING':\n",
    "        progress.value=2\n",
    "        progress.bar_style='info'\n",
    "        progress.description=result['status']\n",
    "    elif result['status']== 'DONE':\n",
    "        progress.value=3\n",
    "        if 'error' in result: \n",
    "            progress.description='Error'\n",
    "            progress.bar_style='danger'\n",
    "        else:\n",
    "            progress.description=result['status']\n",
    "            progress.bar_style='success'\n",
    "\n",
    "import time \n",
    "\n",
    "#repeat until the result is DONE\n",
    "while True:\n",
    "    #obtain the status of the operation\n",
    "    result=compute_service.zoneOperations().get(project=projectId,\n",
    "                                              zone=zone,\n",
    "                                              operation=operation['name']).execute()\n",
    "    updateProgress(result)\n",
    "    if result['status']== 'DONE':\n",
    "        break\n",
    "    time.sleep(.25) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing the instance and their status"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result = compute_service.instances().list(project=projectId, zone=zone).execute()\n",
    "\n",
    "if 'items' in result.keys():\n",
    "    display(DataFrame.from_dict({instance['name']:(instance['status'],'✅'if instance['status']=='RUNNING' else '✖'if instance['status']=='TERMINATED' else '❓')for instance in result['items']},orient='index'))\n",
    "else:\n",
    "    print(\"No instance found.\")\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Start/Stop/Delete instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# getting the current instances list\n",
    "instances=compute_service.instances().list(project=projectId,zone=zone).execute()\n",
    "instancesList=[item['name'] for item in instances['items']]\n",
    "# none is added for the dropdownlist\n",
    "instancesList.append('none')\n",
    "\n",
    "#building and displaying the widgets\n",
    "instancesWidget=widgets.Dropdown(options=instancesList,value='none')\n",
    "instancesValid=widgets.Valid(value=False,description='')\n",
    "instanceAction=widgets.RadioButtons(\n",
    "    options=[ 'Status','Start','Stop', 'Delete'],value='Status')\n",
    "instanceExecute=widgets.ToggleButton(value=False,description='Execute',disabled=True)\n",
    "display(widgets.Box([instancesWidget,instancesValid,instanceAction,instanceExecute]))\n",
    "\n",
    "## execute an operation. \n",
    "def execute(operation):\n",
    "    #exctract the method and the instancename form the operation\n",
    "    instanceName=operation.uri.split('?')[0].split('/')[-1]\n",
    "    methodId=operation.methodId.split('.')[-1]\n",
    "    \n",
    "    #some widgets (action + instance + progress)\n",
    "    progress=widgets.IntProgress(value=0,min=0,max=3,step=1,description=':',bar_style='info')\n",
    "    display(widgets.Box([widgets.Label(value=methodId+\"ing\"),widgets.Label(value=instanceName),progress]))\n",
    "    \n",
    "    #the dropdown and buttons are disabled when an operation is executing\n",
    "    global instanceExecute\n",
    "    global instancesWidget\n",
    "    instancesWidget.disabled=True\n",
    "    instanceExecute.disabled=True\n",
    "    \n",
    "    #execute the operation\n",
    "    operation=operation.execute()\n",
    "    #until the operation is not DONE, we update the progress bar\n",
    "    while True:\n",
    "        result=compute_service.zoneOperations().get(project=projectId,\n",
    "                                              zone=zone,\n",
    "                                              operation=operation['name']).execute()\n",
    "        updateProgress(result,progress)\n",
    "        if result['status']== 'DONE':\n",
    "            if methodId==u'delete':\n",
    "                #when the instance is deleted, it has to be remove from the dropdownlist\n",
    "                global instancesList               \n",
    "                instancesList.remove(instanceName)\n",
    "                instancesWidget.options=instancesList\n",
    "                instancesValid.value=False\n",
    "            #the operation is completed, the dropwdown and buttons are enabled\n",
    "            instancesWidget.disabled=False\n",
    "            instanceExecute.disabled=False\n",
    "            break\n",
    "        time.sleep(0.1) \n",
    "        \n",
    "\n",
    "\n",
    "def executeInstance(sender):\n",
    "    #callback when the execute button is clicked    \n",
    "    if instancesValid.value==True:\n",
    "        # the correct operation is created and pass to the execute method\n",
    "        if instanceAction.value=='Stop':\n",
    "            execute(compute_service.instances().stop(project=projectId,\n",
    "                                  zone=zone,\n",
    "                                  instance=instancesWidget.value\n",
    "                                  ))\n",
    "        elif instanceAction.value=='Start':\n",
    "            execute(compute_service.instances().start(project=projectId,\n",
    "                                  zone=zone,\n",
    "                                  instance=instancesWidget.value\n",
    "                                  ))\n",
    "        elif instanceAction.value=='Delete':\n",
    "            execute(compute_service.instances().delete(project=projectId,\n",
    "                                  zone=zone,\n",
    "                                  instance=instancesWidget.value\n",
    "                                  ))\n",
    "        elif instanceAction.value=='Status': \n",
    "            instance=compute_service.instances().get(project=projectId,\n",
    "                                  zone=zone,\n",
    "                                  instance=instancesWidget.value).execute()\n",
    "            display(widgets.Box([widgets.Label(value=instance['name']),\n",
    "                                 widgets.Label(value=instance['status'])\n",
    "                                ]))\n",
    "def instancesValueChange(sender):\n",
    "    #callback when an element is selected in the dropdown list \n",
    "    if instancesWidget.value!=None:\n",
    "        #when the seleciton is correct the valid widget is valid\n",
    "        instancesValid.value=True\n",
    "        instanceExecute.disabled=False\n",
    "\n",
    "#set up the callback on the widgets        \n",
    "instancesWidget.observe(instancesValueChange, 'value')  \n",
    "instanceExecute.observe(executeInstance,'value')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}