02b_Google_Geo_API_Fetcher.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Retrieve Locations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The purpose of this notebook is to retrieve the locations of the incomplete records. To do so, the Google geo-API is used, it has a limit of 2500 queries per day. Therefore, the following algorithm will have to be able to run mutliple times on the csv containing the locations in order to archieve the retrieval."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# The Geocoder module is used.\n",
    "from pygeocoder import Geocoder\n",
    "import time\n",
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Files\n",
    "input_file = \"./Intermediate Data/Missing_Zip_Lat_Long.csv\"\n",
    "output_file = \"./Intermediate Data/Retrieved_Zip_Lat_Long.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Check if file exists, if not, create it\n",
    "if not os.path.exists(output_file):\n",
    "    open(output_file, 'w').close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def CorrectLongitudeLatitude(n,input_file,output_file):\n",
    "    '''\n",
    "    Goal: retrieve the zipcode, longitude, and latitude of a given pothole.\n",
    "    Input: A csv file containing the missing locations.\n",
    "    Output: A csv file containing the retrieved results.\n",
    "    '''\n",
    "    print(\"Retrieving missing locations using Google geo-API...\")\n",
    "    \n",
    "    # Load dataframe and open the output file\n",
    "    missing_location_df = pd.read_csv(input_file)\n",
    "\n",
    "    # Open the connection with the ouput file as \"read-only\"\n",
    "    output  = open(output_file, 'r')\n",
    "    \n",
    "    # Inspect file to determine how much work is left\n",
    "    n_case_retrieved = output.read().count(\"\\n\")\n",
    "    n_total_cases = missing_location_df.shape[0]\n",
    "\n",
    "    output.close()\n",
    "    output  = open(output_file, 'a')\n",
    "          \n",
    "    print(\"Total cases: \"+str(n_total_cases))\n",
    "    print(\"Work already performed on: \"+str(n_case_retrieved))\n",
    "    \n",
    "    # Set the start and end indexes    \n",
    "    start_index = n_case_retrieved\n",
    "    end_index = min(start_index+n,n_total_cases)\n",
    "    \n",
    "    print(\"Fetch starts on \" + str(min(n,n_total_cases-start_index)) + \" items...\")\n",
    "    \n",
    "    progress=0\n",
    "    for index in range(start_index,end_index):\n",
    "        \n",
    "        current_index = missing_location_df.iloc[index,0]\n",
    "        current_address = missing_location_df.iloc[index,1].replace(\"  \",\" \").strip()\n",
    "        \n",
    "        # If address starts with a 0, remove it\n",
    "        if current_address[0]==\"0\":\n",
    "            current_address = current_address[2:]\n",
    "        \n",
    "        # If address contains two street numbers, keep the second one\n",
    "        if \"-\" in current_address:\n",
    "            current_address = current_address.split(\"-\")[1]\n",
    "            \n",
    "        # If address is an intersection, replace & by and and remove INTERSECTION OF\n",
    "        if \"INTERSECTION of\" in current_address:\n",
    "            current_address = current_address.replace(\"INTERSECTION of \",\"\")\n",
    "            current_address = current_address.replace('&','and')\n",
    "                \n",
    "        time.sleep(0.2)\n",
    "        \n",
    "        try:\n",
    "            location = Geocoder.geocode(current_address)\n",
    "        except:\n",
    "            latitude,longitude,zipcode = \"ERROR\",\"ERROR\",\"ERROR\"\n",
    "        try:\n",
    "            latitude,longitude = location.coordinates\n",
    "        except:\n",
    "            latitude,longitude = \"ERROR\",\"ERROR\"\n",
    "        try:\n",
    "            zipcode = location.postal_code\n",
    "        except:\n",
    "            zipcode = \"ERROR\"\n",
    "        \n",
    "        output_location = str(current_index)+\",\"+str(current_address)+\",\"+str(latitude)+\",\"+str(longitude)+\",\"+str(zipcode)+\"\\n\"\n",
    "\n",
    "        progress+=1\n",
    "        output.write(output_location)\n",
    "        if progress%50==0:\n",
    "            print(\"Done with \"+str(progress)+\" records...\")\n",
    "            \n",
    "    output.close()\n",
    "    print(\"Search completed...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Retrieving missing locations using Google geo-API...\n",
      "Total cases: 5812\n",
      "Work already performed on: 5479\n",
      "Fetch starts on 333 items...\n",
      "Done with 50 records...\n",
      "Done with 100 records...\n",
      "Done with 150 records...\n",
      "Done with 200 records...\n",
      "Done with 250 records...\n",
      "Done with 300 records...\n",
      "Search completed...\n"
     ]
    }
   ],
   "source": [
    "CorrectLongitudeLatitude(500,input_file,output_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "The above function is used several times (by batches of 500 records) in order to avoid triggering the API limitations."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}