02-kita-page-scrapper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandas in /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages (0.25.3)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages (from pandas) (1.18.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages (from pandas) (2019.3)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages (from pandas) (2.8.1)\n",
      "Requirement already satisfied: six>=1.5 in /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages (from python-dateutil>=2.6.1->pandas) (1.13.0)\n",
      "Requirement already satisfied: numpy in /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages (1.18.1)\n",
      "Requirement already satisfied: selenium in /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages (3.141.0)\n",
      "Requirement already satisfied: urllib3 in /Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages (from selenium) (1.24.3)\n"
     ]
    }
   ],
   "source": [
    "!pip3 install pandas\n",
    "!pip3 install numpy\n",
    "!pip3 install selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import time\n",
    "import numpy as np\n",
    "from selenium.webdriver import Chrome\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "\n",
    "# Current directory\n",
    "ROOT_DIR = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>kita_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.kita-navigator.berlin.de/einrichtu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.kita-navigator.berlin.de/einrichtu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.kita-navigator.berlin.de/einrichtu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            kita_url\n",
       "0  https://www.kita-navigator.berlin.de/einrichtu...\n",
       "1  https://www.kita-navigator.berlin.de/einrichtu...\n",
       "2  https://www.kita-navigator.berlin.de/einrichtu..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load the URL of all kitas\n",
    "df_kitas_url = pd.read_csv('df_kitas_url.csv')\n",
    "df_kitas_url.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Main Kita webpage\n",
    "kita_url = 'https://kita-navigator.berlin.de/einrichtungen/363?input=Kolberger%20Str.%2C%2013357%20Berlin%2C%20Alemanha&betb=2-2020&einfacheSuche=false&entfernung=5&lat=52.5455402&lon=13.3773108&seite=1&index=1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List that will store temporarily all \n",
    "# the information about the kita\n",
    "kita_info = []\n",
    "\n",
    "# List that will contain the aufnahmealhalter\n",
    "# information because it's an array of information\n",
    "aufnahmealhalter = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Chrome driver to open the webpage and get the full html object\n",
    "webdriver = ROOT_DIR + \"/chromedriver\"\n",
    "driver = Chrome(webdriver)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The driver will open the Chrome \n",
    "# page and get the contents\n",
    "driver.get(kita_url)\n",
    "\n",
    "# This time will be to wait 2 seconds \n",
    "# to give some time for the entire page load\n",
    "time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Right now we'll get the elements \n",
    "# inside of the page to extract the infos about the Kita\n",
    "\n",
    "# Get the Kita address\n",
    "for elem in driver.find_elements_by_xpath('.//a[@title = \"Externer Link Wegbeschreibung in Google Maps (Öffnet neuen Tab)\"]'):\n",
    "    kita_address = elem.text\n",
    "\n",
    "# Get the Kita website\n",
    "for elem in driver.find_elements_by_xpath('.//a[@title = \"Externer Link Homepage der Kita (Öffnet neuen Tab)\"]'):\n",
    "    kita_website = elem.text\n",
    "    \n",
    "# Get the Kita phone\n",
    "for elem in driver.find_elements_by_xpath('.//a[@title = \"Die Kita anrufen\"]'):\n",
    "    kita_telephone = elem.text\n",
    "\n",
    "# Get the Kita e-mail    \n",
    "for elem in driver.find_elements_by_xpath('.//a[@title = \"Eine E-Mail an die Kita senden\"]'):\n",
    "    kita_email = elem.text\n",
    "\n",
    "# Get the Kita aufnahmealhalter\n",
    "for elem in driver.find_elements_by_xpath('.//span[@class = \"d-block\"]'): \n",
    "    aufnahmealhalter.append(str(elem.text).split(' '))\n",
    "    kita_aufnahmealter = str(elem.text).split(' ')\n",
    "    \n",
    "# Using the Kita aufnahmealhalter we will extract more information\n",
    "kita_distance = ' '.join(aufnahmealhalter[0])\n",
    "kita_aufnahmealter_jahre = ' '.join(aufnahmealhalter[1])\n",
    "kita_maximales_platzangebot = ' '.join(aufnahmealhalter[2])\n",
    "kita_maximales_platzangebot_verfugbar = ' '.join(aufnahmealhalter[3])  \n",
    "\n",
    "\n",
    "# Store everthing in an array using a tuple\n",
    "kita_info.append((kita_address,\n",
    "                  kita_website,\n",
    "                  kita_telephone,\n",
    "                  kita_email,\n",
    "                  kita_aufnahmealter,\n",
    "                  kita_distance,\n",
    "                  kita_aufnahmealter_jahre,\n",
    "                  kita_maximales_platzangebot,\n",
    "                  kita_maximales_platzangebot_verfugbar                  \n",
    "                 ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address</th>\n",
       "      <th>website</th>\n",
       "      <th>phone</th>\n",
       "      <th>email</th>\n",
       "      <th>aufnahmealter</th>\n",
       "      <th>distance</th>\n",
       "      <th>aufnahmealter_jahre</th>\n",
       "      <th>maximales_platzangebot</th>\n",
       "      <th>maximales_platzangebot_verfugbar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Wiesenstr. 25, 13357 Berlin</td>\n",
       "      <td>http://www.unionhilfswerk.de</td>\n",
       "      <td>+49-30-4622076</td>\n",
       "      <td>kita.wiesenstrasse@unionhilfswerk.de</td>\n",
       "      <td>[davon, unter, 3, Jahre:, 30]</td>\n",
       "      <td>0.45 km</td>\n",
       "      <td>3 Monate</td>\n",
       "      <td>87</td>\n",
       "      <td>davon unter 3 Jahre: 30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       address                       website           phone  \\\n",
       "0  Wiesenstr. 25, 13357 Berlin  http://www.unionhilfswerk.de  +49-30-4622076   \n",
       "\n",
       "                                  email                  aufnahmealter  \\\n",
       "0  kita.wiesenstrasse@unionhilfswerk.de  [davon, unter, 3, Jahre:, 30]   \n",
       "\n",
       "  distance aufnahmealter_jahre maximales_platzangebot  \\\n",
       "0  0.45 km            3 Monate                     87   \n",
       "\n",
       "  maximales_platzangebot_verfugbar  \n",
       "0          davon unter 3 Jahre: 30  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Transform in a pandas dataframe just for filtering\n",
    "df = pd.DataFrame(kita_info)\n",
    "\n",
    "# Put the right columns\n",
    "df.columns = ['address',\n",
    "              'website',\n",
    "              'phone',\n",
    "              'email',\n",
    "              'aufnahmealter',\n",
    "              'distance',\n",
    "              'aufnahmealter_jahre',\n",
    "              'maximales_platzangebot',\n",
    "              'maximales_platzangebot_verfugbar' \n",
    "             ]\n",
    "# Check\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above code will run OK if there's all information in website (Phone, website, email, etc), it means that **one of those fields is empty, the solution will break**.\n",
    "\n",
    "In the function below I've included some `try...catch` statements to overcome some of these problems. However the code it'still very fragile yet. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_kita_infos(url_kita):\n",
    "    try:\n",
    "        # List that will store temporarily all the information about the kita\n",
    "        kita_info = []\n",
    "\n",
    "        # List that will contain the aufnahmealhalter information because it's an array of information\n",
    "        aufnahmealhalter = []\n",
    "\n",
    "        # Chrome driver to open the webpage and get the full html object\n",
    "        webdriver = \"/Users/flavio.desouza/Desktop/ml-borracharia/src/kitas-scrapper/chromedriver\"\n",
    "        driver = Chrome(webdriver)\n",
    "\n",
    "        # The driver will open the Chrome page and get the contents\n",
    "        driver.get(url_kita)\n",
    "\n",
    "        # This time will be to wait 2 seconds to give some time for the entire page load\n",
    "        time.sleep(1)\n",
    "\n",
    "        # Right now we'll get the elements inside of the page to extract the infos about the Kita\n",
    "\n",
    "        # Get the Kita address\n",
    "        try:\n",
    "            for elem in driver.find_elements_by_xpath('.//a[@title = \"Externer Link Wegbeschreibung in Google Maps (Öffnet neuen Tab)\"]'):\n",
    "                kita_address = elem.text\n",
    "        except:\n",
    "            kita_address = 'N/A'\n",
    "\n",
    "        # Get the Kita phone\n",
    "        try:    \n",
    "            for elem in driver.find_elements_by_xpath('.//a[@title = \"Die Kita anrufen\"]'):\n",
    "                kita_telephone = elem.text\n",
    "        except:\n",
    "            kita_telephone = 'N/A'\n",
    "\n",
    "        # Get the Kita e-mail\n",
    "        try:    \n",
    "            for elem in driver.find_elements_by_xpath('.//a[@title = \"Eine E-Mail an die Kita senden\"]'):\n",
    "                kita_email = elem.text\n",
    "        except:\n",
    "            kita_email = 'N/A'\n",
    "\n",
    "        # Get the Kita aufnahmealhalter\n",
    "        try:    \n",
    "            for elem in driver.find_elements_by_xpath('.//span[@class = \"d-block\"]'): \n",
    "                aufnahmealhalter.append(str(elem.text).split(' '))\n",
    "                kita_aufnahmealter = str(elem.text).split(' ')\n",
    "        except:\n",
    "            kita_aufnahmealter = 'N/A'            \n",
    "\n",
    "        # Using the Kita aufnahmealhalter we will extract more information\n",
    "        try:\n",
    "            kita_distance = ' '.join(aufnahmealhalter[0])\n",
    "        except:\n",
    "            kita_distance = 'N/A'        \n",
    "\n",
    "        try:\n",
    "            kita_aufnahmealter_jahre = ' '.join(aufnahmealhalter[1])\n",
    "        except:\n",
    "            kita_aufnahmealter_jahre = 'N/A'        \n",
    "\n",
    "        try:\n",
    "            kita_maximales_platzangebot = ' '.join(aufnahmealhalter[2])\n",
    "        except:\n",
    "            kita_maximales_platzangebot = 'N/A'        \n",
    "\n",
    "        try:\n",
    "            kita_maximales_platzangebot_verfugbar = ' '.join(aufnahmealhalter[3])\n",
    "        except:\n",
    "            kita_maximales_platzangebot_verfugbar = 'N/A'        \n",
    "\n",
    "        # Get the Kita website\n",
    "        try:\n",
    "            for elem in driver.find_elements_by_xpath('.//a[@title = \"Externer Link Homepage der Kita (Öffnet neuen Tab)\"]'):\n",
    "                kita_website = elem.text\n",
    "        except:\n",
    "            kita_website = 'N/A'            \n",
    "\n",
    "\n",
    "\n",
    "        # Store everthing in an array using a tuple\n",
    "        kita_info.append((kita_address,\n",
    "                          kita_website,\n",
    "                          kita_telephone,\n",
    "                          kita_email,\n",
    "                          kita_aufnahmealter,\n",
    "                          kita_distance,\n",
    "                          kita_aufnahmealter_jahre,\n",
    "                          kita_maximales_platzangebot,\n",
    "                          kita_maximales_platzangebot_verfugbar                  \n",
    "                         ))\n",
    "\n",
    "        # Transform in a pandas dataframe just for filtering\n",
    "        df = pd.DataFrame(kita_info)\n",
    "\n",
    "        # Put the right columns\n",
    "        df.columns = ['address',\n",
    "                      'website',\n",
    "                      'phone',\n",
    "                      'email',\n",
    "                      'aufnahmealter',\n",
    "                      'distance',\n",
    "                      'aufnahmealter_jahre',\n",
    "                      'maximales_platzangebot',\n",
    "                      'maximales_platzangebot_verfugbar' \n",
    "                     ]\n",
    "        \n",
    "        # Close the window\n",
    "        driver.close()\n",
    "        \n",
    "    except:\n",
    "        print('Invalid kita listing')\n",
    "        \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address</th>\n",
       "      <th>website</th>\n",
       "      <th>phone</th>\n",
       "      <th>email</th>\n",
       "      <th>aufnahmealter</th>\n",
       "      <th>distance</th>\n",
       "      <th>aufnahmealter_jahre</th>\n",
       "      <th>maximales_platzangebot</th>\n",
       "      <th>maximales_platzangebot_verfugbar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Wiesenstr. 25, 13357 Berlin</td>\n",
       "      <td>http://www.unionhilfswerk.de</td>\n",
       "      <td>+49-30-4622076</td>\n",
       "      <td>kita.wiesenstrasse@unionhilfswerk.de</td>\n",
       "      <td>[davon, unter, 3, Jahre:, 30]</td>\n",
       "      <td>0.45 km</td>\n",
       "      <td>3 Monate</td>\n",
       "      <td>87</td>\n",
       "      <td>davon unter 3 Jahre: 30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       address                       website           phone  \\\n",
       "0  Wiesenstr. 25, 13357 Berlin  http://www.unionhilfswerk.de  +49-30-4622076   \n",
       "\n",
       "                                  email                  aufnahmealter  \\\n",
       "0  kita.wiesenstrasse@unionhilfswerk.de  [davon, unter, 3, Jahre:, 30]   \n",
       "\n",
       "  distance aufnahmealter_jahre maximales_platzangebot  \\\n",
       "0  0.45 km            3 Monate                     87   \n",
       "\n",
       "  maximales_platzangebot_verfugbar  \n",
       "0          davon unter 3 Jahre: 30  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Test\n",
    "url_kita = \\\n",
    "    'https://kita-navigator.berlin.de/einrichtungen/363?input=Kolberger%20Str.%2C%2013357%20Berlin%2C%20Alemanha&betb=2-2020&einfacheSuche=false&entfernung=5&lat=52.5455402&lon=13.3773108&seite=1&index=1'\n",
    "\n",
    "df = get_kita_infos(url_kita)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>kita_url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.kita-navigator.berlin.de/einrichtu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.kita-navigator.berlin.de/einrichtu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.kita-navigator.berlin.de/einrichtu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.kita-navigator.berlin.de/einrichtu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.kita-navigator.berlin.de/einrichtu...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            kita_url\n",
       "0  https://www.kita-navigator.berlin.de/einrichtu...\n",
       "1  https://www.kita-navigator.berlin.de/einrichtu...\n",
       "2  https://www.kita-navigator.berlin.de/einrichtu...\n",
       "3  https://www.kita-navigator.berlin.de/einrichtu...\n",
       "4  https://www.kita-navigator.berlin.de/einrichtu..."
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load list of Kitas\n",
    "df_kitas_url = pd.read_csv('df_kitas_url.csv')\n",
    "df_kitas_url.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Invalid kita listing\n",
      "Invalid Kita: https://www.kita-navigator.berlin.de/einrichtungen/742?input=Arnold-Knoblauch-Ring%2C%2014109%20Berlin%2C%20Alemanha&betb=2-2020&einfacheSuche=true&entfernung=5&lat=52.4099966&lon=13.137335&seite=2&index=14\n",
      "Invalid kita listing\n",
      "Invalid Kita: https://www.kita-navigator.berlin.de/einrichtungen/568?input=Arnold-Knoblauch-Ring%2C%2014109%20Berlin%2C%20Alemanha&betb=2-2020&einfacheSuche=true&entfernung=5&lat=52.4099966&lon=13.137335&seite=1&index=0\n",
      "Invalid kita listing\n",
      "Invalid Kita: https://www.kita-navigator.berlin.de/einrichtungen/1823?input=Arnold-Knoblauch-Ring%2C%2014109%20Berlin%2C%20Alemanha&betb=2-2020&einfacheSuche=true&entfernung=5&lat=52.4099966&lon=13.137335&seite=1&index=2\n",
      "Invalid kita listing\n",
      "Invalid Kita: https://www.kita-navigator.berlin.de/einrichtungen/1088?input=Arnold-Knoblauch-Ring%2C%2014109%20Berlin%2C%20Alemanha&betb=2-2020&einfacheSuche=true&entfernung=5&lat=52.4099966&lon=13.137335&seite=2&index=13\n",
      "Invalid kita listing\n",
      "Invalid Kita: https://www.kita-navigator.berlin.de/einrichtungen/1856?input=Arnold-Knoblauch-Ring%2C%2014109%20Berlin%2C%20Alemanha&betb=2-2020&einfacheSuche=true&entfernung=5&lat=52.4099966&lon=13.137335&seite=1&index=4\n",
      "Invalid kita listing\n",
      "Invalid Kita: https://www.kita-navigator.berlin.de/einrichtungen/1855?input=Arnold-Knoblauch-Ring%2C%2014109%20Berlin%2C%20Alemanha&betb=2-2020&einfacheSuche=true&entfernung=5&lat=52.4099966&lon=13.137335&seite=1&index=1\n",
      "Invalid kita listing\n",
      "Invalid Kita: https://www.kita-navigator.berlin.de/einrichtungen/928?input=Arnold-Knoblauch-Ring%2C%2014109%20Berlin%2C%20Alemanha&betb=2-2020&einfacheSuche=true&entfernung=5&lat=52.4099966&lon=13.137335&seite=1&index=8\n",
      "Invalid kita listing\n",
      "Invalid Kita: https://www.kita-navigator.berlin.de/einrichtungen/2883?input=Arnold-Knoblauch-Ring%2C%2014109%20Berlin%2C%20Alemanha&betb=2-2020&einfacheSuche=true&entfernung=5&lat=52.4099966&lon=13.137335&seite=1&index=3\n"
     ]
    }
   ],
   "source": [
    "# Empty pandas dataframe for store all infos\n",
    "df_kitas_information = pd.DataFrame()\n",
    "\n",
    "for index, row in df_kitas_url.iterrows():\n",
    "    \n",
    "    # Get th information from the URL using our function and store in a pandas dataframe\n",
    "    try:\n",
    "        df = get_kita_infos(row.kita_url)\n",
    "    \n",
    "        # Salve the record inside our dataframe that we will use for analysis\n",
    "        df_kitas_information = df_kitas_information.append(df)\n",
    "        \n",
    "    except:\n",
    "        print(f'Invalid Kita: {row.kita_url}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>address</th>\n",
       "      <th>website</th>\n",
       "      <th>phone</th>\n",
       "      <th>email</th>\n",
       "      <th>aufnahmealter</th>\n",
       "      <th>distance</th>\n",
       "      <th>aufnahmealter_jahre</th>\n",
       "      <th>maximales_platzangebot</th>\n",
       "      <th>maximales_platzangebot_verfugbar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Zum Heckeshorn 33, 14109 Berlin</td>\n",
       "      <td>http://www.berliner-waldkindergarten.de</td>\n",
       "      <td>+49-176-81751568</td>\n",
       "      <td>info@berliner-waldkindergarten.de</td>\n",
       "      <td>[davon, unter, 3, Jahre:, 3]</td>\n",
       "      <td>2.8 km</td>\n",
       "      <td>24 Monate</td>\n",
       "      <td>21</td>\n",
       "      <td>davon unter 3 Jahre: 3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sakrower Landstr. 56, 14089 Berlin</td>\n",
       "      <td>http://www.kita-mariaehimmelfahrt.cfj-caritas-...</td>\n",
       "      <td>+49-30-3656021</td>\n",
       "      <td>kita-mariaehimmelfahrt@t-online.de</td>\n",
       "      <td>[davon, unter, 3, Jahre:, 25]</td>\n",
       "      <td>4.63 km</td>\n",
       "      <td>12 Monate</td>\n",
       "      <td>84</td>\n",
       "      <td>davon unter 3 Jahre: 25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Zum Heckeshorn 40, 14109 Berlin</td>\n",
       "      <td>http://www.montessori-friends.de/kinderhaus-am...</td>\n",
       "      <td>+49-30-23326400</td>\n",
       "      <td>kita@montessori-friends.de</td>\n",
       "      <td>[davon, unter, 3, Jahre:, 10]</td>\n",
       "      <td>2.88 km</td>\n",
       "      <td>24 Monate</td>\n",
       "      <td>30</td>\n",
       "      <td>davon unter 3 Jahre: 10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Königstr. 35C, 14109 Berlin</td>\n",
       "      <td>http://www.jaegerhof-berlin.de</td>\n",
       "      <td>+49-30-80587121</td>\n",
       "      <td>info@jaegerhof-berlin.de</td>\n",
       "      <td>[davon, unter, 3, Jahre:, 20]</td>\n",
       "      <td>2.53 km</td>\n",
       "      <td>12 Monate</td>\n",
       "      <td>55</td>\n",
       "      <td>davon unter 3 Jahre: 20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Von-der-Trenck-Str. 4, 14129 Berlin</td>\n",
       "      <td>http://www.tandembtl.de</td>\n",
       "      <td>+49-30-80402003</td>\n",
       "      <td>kita-zak@tandembtl.de</td>\n",
       "      <td>[90]</td>\n",
       "      <td>4.55 km</td>\n",
       "      <td>12 Monate</td>\n",
       "      <td>90</td>\n",
       "      <td>N/A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Normannenstr. 1, 14129 Berlin</td>\n",
       "      <td>http://www.giardino-dei-bambini.de</td>\n",
       "      <td>+49-30-36752100</td>\n",
       "      <td>kontakt@giardino-dei-bambini.de</td>\n",
       "      <td>[davon, unter, 3, Jahre:, 1]</td>\n",
       "      <td>4.56 km</td>\n",
       "      <td>30 Monate</td>\n",
       "      <td>20</td>\n",
       "      <td>davon unter 3 Jahre: 1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Gößweinsteiner Gang 21, 14089 Berlin</td>\n",
       "      <td>http://www.kita-traumhaus-kladow.de</td>\n",
       "      <td>+49-30-3657458</td>\n",
       "      <td>info@kita-traumhaus-kladow.de</td>\n",
       "      <td>[davon, unter, 3, Jahre:, 13]</td>\n",
       "      <td>4.4 km</td>\n",
       "      <td>12 Monate</td>\n",
       "      <td>53</td>\n",
       "      <td>davon unter 3 Jahre: 13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                address  \\\n",
       "0       Zum Heckeshorn 33, 14109 Berlin   \n",
       "0    Sakrower Landstr. 56, 14089 Berlin   \n",
       "0       Zum Heckeshorn 40, 14109 Berlin   \n",
       "0           Königstr. 35C, 14109 Berlin   \n",
       "0   Von-der-Trenck-Str. 4, 14129 Berlin   \n",
       "0         Normannenstr. 1, 14129 Berlin   \n",
       "0  Gößweinsteiner Gang 21, 14089 Berlin   \n",
       "\n",
       "                                             website             phone  \\\n",
       "0            http://www.berliner-waldkindergarten.de  +49-176-81751568   \n",
       "0  http://www.kita-mariaehimmelfahrt.cfj-caritas-...    +49-30-3656021   \n",
       "0  http://www.montessori-friends.de/kinderhaus-am...   +49-30-23326400   \n",
       "0                     http://www.jaegerhof-berlin.de   +49-30-80587121   \n",
       "0                            http://www.tandembtl.de   +49-30-80402003   \n",
       "0                 http://www.giardino-dei-bambini.de   +49-30-36752100   \n",
       "0                http://www.kita-traumhaus-kladow.de    +49-30-3657458   \n",
       "\n",
       "                                email                  aufnahmealter distance  \\\n",
       "0   info@berliner-waldkindergarten.de   [davon, unter, 3, Jahre:, 3]   2.8 km   \n",
       "0  kita-mariaehimmelfahrt@t-online.de  [davon, unter, 3, Jahre:, 25]  4.63 km   \n",
       "0          kita@montessori-friends.de  [davon, unter, 3, Jahre:, 10]  2.88 km   \n",
       "0            info@jaegerhof-berlin.de  [davon, unter, 3, Jahre:, 20]  2.53 km   \n",
       "0               kita-zak@tandembtl.de                           [90]  4.55 km   \n",
       "0     kontakt@giardino-dei-bambini.de   [davon, unter, 3, Jahre:, 1]  4.56 km   \n",
       "0       info@kita-traumhaus-kladow.de  [davon, unter, 3, Jahre:, 13]   4.4 km   \n",
       "\n",
       "  aufnahmealter_jahre maximales_platzangebot maximales_platzangebot_verfugbar  \n",
       "0           24 Monate                     21           davon unter 3 Jahre: 3  \n",
       "0           12 Monate                     84          davon unter 3 Jahre: 25  \n",
       "0           24 Monate                     30          davon unter 3 Jahre: 10  \n",
       "0           12 Monate                     55          davon unter 3 Jahre: 20  \n",
       "0           12 Monate                     90                              N/A  \n",
       "0           30 Monate                     20           davon unter 3 Jahre: 1  \n",
       "0           12 Monate                     53          davon unter 3 Jahre: 13  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_kitas_information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Several Kitas that doesn't contains some info have been leftover. This will be fixed soon."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_kitas_information.to_csv('df_kitas_information.csv', index=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}