NZ_Covid_19_Data_Dump_June_2020_Version_Public.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NZ Covid 19 Data Dump - June 2020 Version - Public.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyP/IVOfbUTrQBZNU/oPdYn5",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ernestkung/NZCovidDocs/blob/master/NZ_Covid_19_Data_Dump_June_2020_Version_Public.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3eBNlp2BwUqg",
        "colab_type": "text"
      },
      "source": [
        "Improved version for the June 2020 document dump. Builds on the May 2020 version.\n",
        "\n",
        "Adds MySQL database to be able to track what was already downloaded last time. \n",
        "\n",
        "MD5 fingerprint is created for each document to detect matching documents.\n",
        "\n",
        "Main source of documents: https://uniteforrecovery.govt.nz/updates-and-resources/legislation-and-key-documents/proactive-release/"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0YyW8kzBIkOg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "a3b50c7d-34ed-4e99-a0fb-e6deb9f2f1b4"
      },
      "source": [
        "# Need the standard MySQL connector in Python\n",
        "!pip install mysql-connector-python"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting mysql-connector-python\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d1/53/4cf90d2fe81b9cdb55dc180951bcec44ea8685665f1bdb1412501dc362dd/mysql_connector_python-8.0.20-cp36-cp36m-manylinux1_x86_64.whl (14.8MB)\n",
            "\u001b[K     |████████████████████████████████| 14.8MB 289kB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from mysql-connector-python) (3.10.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.0.0->mysql-connector-python) (47.3.1)\n",
            "Requirement already satisfied: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.0.0->mysql-connector-python) (1.12.0)\n",
            "Installing collected packages: mysql-connector-python\n",
            "Successfully installed mysql-connector-python-8.0.20\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nVm0zYZxhe6H",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "8a67ca47-b783-42a9-b728-2a0737ad0d3e"
      },
      "source": [
        "# HERE'S THE PRODUCTION VERSION\n",
        "\n",
        "from urllib import request\n",
        "from bs4 import BeautifulSoup\n",
        "import copy\n",
        "import requests\n",
        "import mysql.connector\n",
        "from mysql.connector import errorcode\n",
        "from mysql.connector.constants import ClientFlag\n",
        "import datetime as dt\n",
        "from dateutil import tz\n",
        "import hashlib\n",
        "\n",
        "def open_mysql_connection ():\n",
        "  \"\"\"Using MySQL settings to open connection to Google Cloud SQL service. Returns a MySQL connection object.\"\"\"\n",
        "  mysql_user = \"\"\n",
        "  mysql_pwd = \"\"\n",
        "  mysql_host = \"\"\n",
        "  ssl_ca_file = \"\"\n",
        "  ssl_key_file = \"\"\n",
        "  ssl_cert_file = \"\"\n",
        "\n",
        "  config = {\n",
        "      'user': mysql_user,\n",
        "      'password': mysql_pwd,\n",
        "      'host': mysql_host,\n",
        "      'client_flags': [ClientFlag.SSL],\n",
        "      'ssl_ca': ssl_ca_file,\n",
        "      'ssl_cert': ssl_cert_file,\n",
        "      'ssl_key': ssl_key_file,\n",
        "  }\n",
        "\n",
        "  try:\n",
        "    cnx = mysql.connector.connect(**config)\n",
        "  except mysql.connector.Error as err:\n",
        "    if err.errno == errorcode.ER_ACCESS_DENIED_ERROR:\n",
        "      return \"Something is wrong with your user name or password\"\n",
        "    elif err.errno == errorcode.ER_BAD_DB_ERROR:\n",
        "      return \"Database does not exist\"\n",
        "    else:\n",
        "      return err\n",
        "  else:\n",
        "    return cnx\n",
        "\n",
        "def convert_dt_to_mysql (\n",
        "    input_datetime  \n",
        "    ):\n",
        "    \"\"\" Use this to convert a datetime value to UTC and into a format suitable for saving in MySQL. Requires datetime, dateutil tz to be imported. \"\"\" \n",
        "    utc_dt = input_datetime.astimezone(tz.gettz(\"Etc/UTC\"))     \n",
        "    mysql_dt = f\"{utc_dt:%Y-%m-%d %H:%M:%S.%f}\"\n",
        "    return mysql_dt\n",
        "\n",
        "download_directory = \"/content/drive/My Drive/downloads2/wage_subsidy_and_leave_schemes/\" # CHECK YOUR LOCAL DOWNLOAD DIRECTORY LOCATION\n",
        "release_url = \"https://uniteforrecovery.govt.nz/updates-and-resources/legislation-and-key-documents/proactive-release/wage-subsidy-and-leave-schemes/\" # CHECK THIS LINK. CHANGE PER DOWNLOAD PAGE.\n",
        "page_data = request.urlopen(release_url)\n",
        "the_page = page_data.read()\n",
        "\n",
        "chowder = BeautifulSoup(the_page, \"html.parser\")\n",
        "\n",
        "bisque_documents = chowder.find_all(\"div\", class_=\"content-element__content\") # This is a list of each section of the page with the noted div + class tags\n",
        "\n",
        "extracted_link = None\n",
        "combined_link = None\n",
        "extracted_title = None\n",
        "extracted_dt = None\n",
        "combined_date_time = None\n",
        "extracted_type = None\n",
        "current_section = 1\n",
        "\n",
        "tz_auckland = tz.gettz(\"Pacific/Auckland\") # This creates the timezone object to be used in a datetime Constructor.\n",
        "\n",
        "for section in bisque_documents:  # Loops through each section of the page\n",
        "\n",
        "  print (\"Parsing occurence of the Div Class:\", current_section)\n",
        "  current_section = current_section + 1\n",
        "\n",
        "  bisque_copy = copy.copy(section)\n",
        "\n",
        "  bisque2_documents = bisque_copy.find_all(\"tr\") # This is a list of each line with \"td\"\n",
        "  print(\"Number of TR rows found:\", len(bisque2_documents))\n",
        "\n",
        "  for section2 in bisque2_documents:  # Loops through each instance of the \"tr\" tag\n",
        "\n",
        "    bisque2_copy = copy.copy(section2)\n",
        "    bisque3_documents = bisque2_copy.find_all(\"td\")  \n",
        "\n",
        "    print(\"Number of TD columns found:\", len(bisque3_documents))\n",
        "    \n",
        "    if len(bisque3_documents) != 3: # Continue on to next loop because not having 3 TD columns means this is the header row.\n",
        "      continue\n",
        "    \n",
        "    td_column_number = 1\n",
        "    break_this_loop = 0\n",
        "\n",
        "    for current_line in bisque3_documents:\n",
        "      print (\"Processing Column:\", td_column_number)\n",
        "      print (current_line)\n",
        "     \n",
        "      if td_column_number == 1:\n",
        "        try:\n",
        "          find_link = current_line.find(\"a\")\n",
        "          extracted_link = find_link.get(\"href\")\n",
        "        except:\n",
        "          break_this_loop = 1\n",
        "          continue\n",
        "        else:\n",
        "          pass\n",
        "        \n",
        "        start_of_link = \"https://uniteforrecovery.govt.nz\" # WATCH OUT FOR CHANGES IN THE DOMAIN NAME. IT ALREADY CHANGED ONCE FROM COVID19.GOVT.NZ.\n",
        "        combined_link = start_of_link + extracted_link\n",
        "\n",
        "      td_column_number = td_column_number + 1 \n",
        "      # This loop section has been set up to accomodate future processing of the other available fields such as document title, document date, document type.\n",
        "    \n",
        "    if break_this_loop == 1:    # This was added because some pages use \"td\" for header rows instead of \"th\"\n",
        "      continue\n",
        "      \n",
        "    print (\"Link to Download:\", combined_link)\n",
        "    r = requests.get(combined_link)\n",
        "    md5_r_file = hashlib.md5(r.content)   # The new innovation in this version is the generation of a MD5 fingerprint\n",
        "    md5_r_file_digest = md5_r_file.hexdigest()\n",
        "    print(\"MD5 of File on the Internet is: \", md5_r_file_digest)\n",
        "\n",
        "    cnx = open_mysql_connection() # This calls a custom function that has the configuration info for MySQL service. Returns a MySQL connection object.\n",
        "    cursor = cnx.cursor() \n",
        "\n",
        "    sql_code = f\"\"\"\n",
        "    SELECT url_of_download FROM covid19docs.files \n",
        "    WHERE doc_hash = '{md5_r_file_digest}'\n",
        "    LIMIT 1\n",
        "    \"\"\"\n",
        "    # Matches the MD5 hash of the Internet file to the database to look for a match. If there isn't a match, the file will be downloaded.\n",
        "    print (sql_code)\n",
        "\n",
        "    cursor.execute(sql_code)\n",
        "\n",
        "    content = cursor.fetchone()\n",
        "    print (content)\n",
        "\n",
        "    if content == None:\n",
        "      utc_now = dt.datetime.now(tz.gettz(\"Etc/UTC\")) \n",
        "      utc_now_mysql = convert_dt_to_mysql(utc_now)\n",
        "\n",
        "      # Insert results to database. The database has a few more columns but they are not used right now.\n",
        "      sql_code = f\"\"\"\n",
        "      INSERT INTO covid19docs.files (doc_type_future, downloaded_dt_utc, url_of_download, added_by_user_id, doc_hash)\n",
        "      VALUES ('1', '{utc_now_mysql}', '{combined_link}', '1', '{md5_r_file_digest}');\n",
        "      \"\"\"\n",
        "      print (sql_code)\n",
        "\n",
        "      try:\n",
        "        cursor.execute(sql_code)\n",
        "        \n",
        "        file_name_start_index = combined_link.rfind(\"/\")\n",
        "        line_length = len(combined_link)\n",
        "        file_name_iso = combined_link[(file_name_start_index+1):line_length]\n",
        "        local_file_name = download_directory + file_name_iso\n",
        "\n",
        "        try:\n",
        "          with open(local_file_name, \"xb\") as f: \n",
        "            f.write(r.content)\n",
        "            print(\"Saved file locally as:\", local_file_name)\n",
        "        except FileExistsError:\n",
        "          print(\"File name already exists in Google Drive\")\n",
        "        else:\n",
        "          pass      \n",
        "\n",
        "      except Exception as e:\n",
        "        cnx.rollback() # Rolls back the insert/updates to the database if anything goes wrong. \n",
        "        print(e)\n",
        "      else:\n",
        "        cnx.commit() # Commits the inserts/updates to the database. \n",
        "\n",
        "    if content != None:\n",
        "      print (\"The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\")\n",
        " \n",
        "      for x in content:\n",
        "        print (x)\n",
        "        \n",
        "        if x != combined_link:\n",
        "          sql_code = f\"\"\"\n",
        "          UPDATE covid19docs.files \n",
        "          SET url_of_download_2 = '{x}'\n",
        "          WHERE doc_hash = '{md5_r_file_digest}';\n",
        "          \"\"\"\n",
        "          print (sql_code)\n",
        "  \n",
        "          try:\n",
        "            cursor.execute(sql_code)\n",
        "        \n",
        "          except Exception as e:\n",
        "            cnx.rollback() \n",
        "            print(e)\n",
        "          else:\n",
        "            cnx.commit() \n",
        "\n",
        "    cursor.close() \n",
        "    cnx.close() \n",
        "\n",
        "### ENDS ###\n",
        "      "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Parsing occurence of the Div Class: 1\n",
            "Number of TR rows found: 15\n",
            "Number of TD columns found: 0\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/COVID-19-How-self-isolating-workers-are-treated-under-workplace-law.pdf\">COVID-19 - How self-isolating workers are treated under workplace law<span class=\"fileExt\"> [PDF, 5.1 MB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>4 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Briefing</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/COVID-19-How-self-isolating-workers-are-treated-under-workplace-law.pdf\n",
            "MD5 of File on the Internet is:  8bd7e37152514044a48a2b725fa8ad5c\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '8bd7e37152514044a48a2b725fa8ad5c'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/COVID-19-How-self-isolating-workers-are-treated-under-workplace-law.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/COVID-19-How-self-isolating-workers-are-treated-under-workplace-law.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Aide-Memoire-Wage-Subsidy-Scheme-Note.pdf\">Aide Memoire: Wage subsidy scheme note<span class=\"fileExt\"> [PDF, 743 KB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>14 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Advice</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Aide-Memoire-Wage-Subsidy-Scheme-Note.pdf\n",
            "MD5 of File on the Internet is:  7385bde8eb442b7fd3e27101a52768f3\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '7385bde8eb442b7fd3e27101a52768f3'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Aide-Memoire-Wage-Subsidy-Scheme-Note.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Aide-Memoire-Wage-Subsidy-Scheme-Note.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/COVID-19-Financial-Support-Paper-B-COVID-19-Leave-Payment-Scheme.pdf\">COVID-19 Financial Support Paper B - COVID-19 Leave Payment Scheme (CAB-20-MIN-0105)<span class=\"fileExt\"> [PDF, 943 KB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>16 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Minute and Paper</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/COVID-19-Financial-Support-Paper-B-COVID-19-Leave-Payment-Scheme.pdf\n",
            "MD5 of File on the Internet is:  935ec0d157485605229ab1a2481b524a\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '935ec0d157485605229ab1a2481b524a'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/COVID-19-Financial-Support-Paper-B-COVID-19-Leave-Payment-Scheme.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/COVID-19-Financial-Support-Paper-B-COVID-19-Leave-Payment-Scheme.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Draft-Cabinet-paper-COVID-19-Order-in-Council-to-exclude-the-Wage-Subsidy-and-Leave-Payment-from-the-application-of-GST.pdf\">Draft Cabinet paper – COVID-19 – Order in Council to exclude the Wage Subsidy and Leave Payment from the application of GST<span class=\"fileExt\"> [PDF, 1.1 MB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>17 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Report</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Draft-Cabinet-paper-COVID-19-Order-in-Council-to-exclude-the-Wage-Subsidy-and-Leave-Payment-from-the-application-of-GST.pdf\n",
            "MD5 of File on the Internet is:  cec920fa44aedb7de2f6e71947ef55fb\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = 'cec920fa44aedb7de2f6e71947ef55fb'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Draft-Cabinet-paper-COVID-19-Order-in-Council-to-exclude-the-Wage-Subsidy-and-Leave-Payment-from-the-application-of-GST.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Draft-Cabinet-paper-COVID-19-Order-in-Council-to-exclude-the-Wage-Subsidy-and-Leave-Payment-from-the-application-of-GST.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Expanding-the-COVID-19-Wage-Subsidy-Scheme-and-Adapting-it-to-Support-Furloughed-Workers-Minute.pdf\">Expanding the COVID-19 Wage Subsidy Scheme and Adapting it to Support Furloughed Workers (CAB-20-MIN-0134)<span class=\"fileExt\"> [PDF, 783 KB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>23 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Minute</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Expanding-the-COVID-19-Wage-Subsidy-Scheme-and-Adapting-it-to-Support-Furloughed-Workers-Minute.pdf\n",
            "MD5 of File on the Internet is:  017991fe08f97a69aff4b95cdd9e8b03\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '017991fe08f97a69aff4b95cdd9e8b03'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Expanding-the-COVID-19-Wage-Subsidy-Scheme-and-Adapting-it-to-Support-Furloughed-Workers-Minute.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Expanding-the-COVID-19-Wage-Subsidy-Scheme-and-Adapting-it-to-Support-Furloughed-Workers-Minute.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Expanding-the-COVID-19-Wage-Subsidy-Scheme-and-Adapting-it-to-Support-Furloughed-Workers-Paper.pdf\">Expanding the COVID-19 Wage Subsidy Scheme and Adapting it to Support Furloughed Workers (CAB-20-SUB-134)<span class=\"fileExt\"> [PDF, 819 KB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>23 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Paper</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Expanding-the-COVID-19-Wage-Subsidy-Scheme-and-Adapting-it-to-Support-Furloughed-Workers-Paper.pdf\n",
            "MD5 of File on the Internet is:  c7c1b51e06214cfeb8aa39ecaad974e7\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = 'c7c1b51e06214cfeb8aa39ecaad974e7'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Expanding-the-COVID-19-Wage-Subsidy-Scheme-and-Adapting-it-to-Support-Furloughed-Workers-Paper.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Expanding-the-COVID-19-Wage-Subsidy-Scheme-and-Adapting-it-to-Support-Furloughed-Workers-Paper.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Working-for-Families-Tax-credits-entitlement-for-Emergency-Benefit-recipients.pdf\">Working for Families Tax Credits entitlement for Emergency Benefit recipients<span class=\"fileExt\"> [PDF, 1.6 MB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>24 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Report</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Working-for-Families-Tax-credits-entitlement-for-Emergency-Benefit-recipients.pdf\n",
            "MD5 of File on the Internet is:  3d61348bd2b730e251bf9cf4878c240c\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '3d61348bd2b730e251bf9cf4878c240c'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Working-for-Families-Tax-credits-entitlement-for-Emergency-Benefit-recipients.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Working-for-Families-Tax-credits-entitlement-for-Emergency-Benefit-recipients.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Final-Cabinet-Paper-and-Advice-on-Enhanced-Wage-Subsidy.pdf\">Final Cabinet Paper and Advice on Enhanced Wage Subsidy<span class=\"fileExt\"> [PDF, 776 KB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>25 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Advice</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Final-Cabinet-Paper-and-Advice-on-Enhanced-Wage-Subsidy.pdf\n",
            "MD5 of File on the Internet is:  0ba103924ff2f715cde286516763b24b\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '0ba103924ff2f715cde286516763b24b'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Final-Cabinet-Paper-and-Advice-on-Enhanced-Wage-Subsidy.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Final-Cabinet-Paper-and-Advice-on-Enhanced-Wage-Subsidy.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Updated-design-settings-for-the-wage-subsidy-scheme.pdf\">Updated design settings for the Wage Subsidy Scheme<span class=\"fileExt\"> [PDF, 1.4 MB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>26 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Advice</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Updated-design-settings-for-the-wage-subsidy-scheme.pdf\n",
            "MD5 of File on the Internet is:  ca667d1fa573b983950b2691a5d2e00a\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = 'ca667d1fa573b983950b2691a5d2e00a'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Updated-design-settings-for-the-wage-subsidy-scheme.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Updated-design-settings-for-the-wage-subsidy-scheme.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Transitioning-to-an-Enhanced-Wage-Subsidy-Scheme-Paper-and-Minute.pdf\">Transitioning to an Enhanced Wage Subsidy Scheme<span class=\"fileExt\"> [PDF, 1.2 MB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>26 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Minute and Paper</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Transitioning-to-an-Enhanced-Wage-Subsidy-Scheme-Paper-and-Minute.pdf\n",
            "MD5 of File on the Internet is:  4a0398f8bcfdc055e4df8b00eb94bdda\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '4a0398f8bcfdc055e4df8b00eb94bdda'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Transitioning-to-an-Enhanced-Wage-Subsidy-Scheme-Paper-and-Minute.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Transitioning-to-an-Enhanced-Wage-Subsidy-Scheme-Paper-and-Minute.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Administrative-Modifications-to-Wage-Subsidy-Minute.pdf\">Administrative Modifications to Wage Subsidy<span class=\"fileExt\"> [PDF, 830 KB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>28 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Minute</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Administrative-Modifications-to-Wage-Subsidy-Minute.pdf\n",
            "MD5 of File on the Internet is:  44d8f0bf2ebb5ff138bc89af0b9336c0\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '44d8f0bf2ebb5ff138bc89af0b9336c0'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Administrative-Modifications-to-Wage-Subsidy-Minute.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Administrative-Modifications-to-Wage-Subsidy-Minute.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Essential-Service-Workers-COVID-19-Leave-Scheme.pdf\">Essential Service Workers COVID-19 Leave Scheme<span class=\"fileExt\"> [PDF, 1.3 MB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>29 March 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Advice</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Essential-Service-Workers-COVID-19-Leave-Scheme.pdf\n",
            "MD5 of File on the Internet is:  bc9f79bbed5a2b64d0e5b599e8645a08\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = 'bc9f79bbed5a2b64d0e5b599e8645a08'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Essential-Service-Workers-COVID-19-Leave-Scheme.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Essential-Service-Workers-COVID-19-Leave-Scheme.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Establishing-an-Essential-Workers-Leave-Support-Scheme-Minute.pdf\">Establishing an Essential Workers Leave Support Scheme (CBC-20-MIN-0020)<span class=\"fileExt\"> [PDF, 791 KB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>1 April 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Minute</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Establishing-an-Essential-Workers-Leave-Support-Scheme-Minute.pdf\n",
            "MD5 of File on the Internet is:  0dc4bd44201a2f3a9b49723581d93926\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '0dc4bd44201a2f3a9b49723581d93926'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Establishing-an-Essential-Workers-Leave-Support-Scheme-Minute.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Establishing-an-Essential-Workers-Leave-Support-Scheme-Minute.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release/Establishing-an-Essential-Workers-Leave-Support-Scheme-Paper.pdf\">Establishing an Essential Workers Leave Support Scheme<span class=\"fileExt\"> [PDF, 1.2 MB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>1 April 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Paper</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Establishing-an-Essential-Workers-Leave-Support-Scheme-Paper.pdf\n",
            "MD5 of File on the Internet is:  42e7e65d39229f1fa5bb5323925ab47f\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '42e7e65d39229f1fa5bb5323925ab47f'\n",
            "    LIMIT 1\n",
            "    \n",
            "('https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Establishing-an-Essential-Workers-Leave-Support-Scheme-Paper.pdf',)\n",
            "The government previously released this file. It has the same MD5 signature as a file already in the MySQL database.\n",
            "https://uniteforrecovery.govt.nz/assets/resources/proactive-release/Establishing-an-Essential-Workers-Leave-Support-Scheme-Paper.pdf\n",
            "Parsing occurence of the Div Class: 2\n",
            "Number of TR rows found: 7\n",
            "Number of TD columns found: 0\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release-2020-june/Joint-Report-Clarification-on-eligibility-for-the-current-COVID-19-wage-subsidy-and-leave-schemes-the-Schemes.pdf\">Clarification on eligibility for the current COVID-19 wage subsidy and leave schemes (the Schemes) and the implementation approach<span class=\"fileExt\"> [PDF, 2.2 MB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>17 April 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Joint report</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/Joint-Report-Clarification-on-eligibility-for-the-current-COVID-19-wage-subsidy-and-leave-schemes-the-Schemes.pdf\n",
            "MD5 of File on the Internet is:  ca2efc5ca25bbd97e5afb71627b75cdb\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = 'ca2efc5ca25bbd97e5afb71627b75cdb'\n",
            "    LIMIT 1\n",
            "    \n",
            "None\n",
            "\n",
            "      INSERT INTO covid19docs.files (doc_type_future, downloaded_dt_utc, url_of_download, added_by_user_id, doc_hash)\n",
            "      VALUES ('1', '2020-06-25 21:57:37.073528', 'https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/Joint-Report-Clarification-on-eligibility-for-the-current-COVID-19-wage-subsidy-and-leave-schemes-the-Schemes.pdf', '1', 'ca2efc5ca25bbd97e5afb71627b75cdb');\n",
            "      \n",
            "Saved file locally as: /content/drive/My Drive/downloads2/wage_subsidy_and_leave_schemes/Joint-Report-Clarification-on-eligibility-for-the-current-COVID-19-wage-subsidy-and-leave-schemes-the-Schemes.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release-2020-june/PAPER-Broadening-Access-to-the-COVID-19-Leave-Payment-Scheme.pdf\">Broadening Access to the COVID-19 Leave Payment Scheme<span class=\"fileExt\"> [PDF, 793 KB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>24 April 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Paper</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/PAPER-Broadening-Access-to-the-COVID-19-Leave-Payment-Scheme.pdf\n",
            "MD5 of File on the Internet is:  649866993110ffe50eef13a8c0933581\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '649866993110ffe50eef13a8c0933581'\n",
            "    LIMIT 1\n",
            "    \n",
            "None\n",
            "\n",
            "      INSERT INTO covid19docs.files (doc_type_future, downloaded_dt_utc, url_of_download, added_by_user_id, doc_hash)\n",
            "      VALUES ('1', '2020-06-25 21:57:37.924988', 'https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/PAPER-Broadening-Access-to-the-COVID-19-Leave-Payment-Scheme.pdf', '1', '649866993110ffe50eef13a8c0933581');\n",
            "      \n",
            "Saved file locally as: /content/drive/My Drive/downloads2/wage_subsidy_and_leave_schemes/PAPER-Broadening-Access-to-the-COVID-19-Leave-Payment-Scheme.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release-2020-june/Report-Addendum-Clarification-on-eligibility-for-the-current-COVID-19-wage-subsidy-and-leave-schemes-the-Schemes-and-the-implementation-approach.pdf\">Addendum to Clarification on eligibility for the current COVID-19 wage subsidy and leave schemes (the Schemes) and the implementation approach<span class=\"fileExt\"> [PDF, 470 KB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>24 April 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Report</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/Report-Addendum-Clarification-on-eligibility-for-the-current-COVID-19-wage-subsidy-and-leave-schemes-the-Schemes-and-the-implementation-approach.pdf\n",
            "MD5 of File on the Internet is:  67e7d0a08316dbb167ab968d17edf3e6\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '67e7d0a08316dbb167ab968d17edf3e6'\n",
            "    LIMIT 1\n",
            "    \n",
            "None\n",
            "\n",
            "      INSERT INTO covid19docs.files (doc_type_future, downloaded_dt_utc, url_of_download, added_by_user_id, doc_hash)\n",
            "      VALUES ('1', '2020-06-25 21:57:38.759890', 'https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/Report-Addendum-Clarification-on-eligibility-for-the-current-COVID-19-wage-subsidy-and-leave-schemes-the-Schemes-and-the-implementation-approach.pdf', '1', '67e7d0a08316dbb167ab968d17edf3e6');\n",
            "      \n",
            "Saved file locally as: /content/drive/My Drive/downloads2/wage_subsidy_and_leave_schemes/Report-Addendum-Clarification-on-eligibility-for-the-current-COVID-19-wage-subsidy-and-leave-schemes-the-Schemes-and-the-implementation-approach.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release-2020-june/BRIEFING-Policy-decisions-on-aspects-of-the-COVID-19-Leave-Support-Scheme.pdf\">Policy decisions on aspects of the COVID-19 Leave Support Scheme <span class=\"fileExt\"> [PDF, 1.1 MB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>25 April 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Briefing</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/BRIEFING-Policy-decisions-on-aspects-of-the-COVID-19-Leave-Support-Scheme.pdf\n",
            "MD5 of File on the Internet is:  85da5a1f35c3b33fd81c36c6c6fd0419\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '85da5a1f35c3b33fd81c36c6c6fd0419'\n",
            "    LIMIT 1\n",
            "    \n",
            "None\n",
            "\n",
            "      INSERT INTO covid19docs.files (doc_type_future, downloaded_dt_utc, url_of_download, added_by_user_id, doc_hash)\n",
            "      VALUES ('1', '2020-06-25 21:57:39.618146', 'https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/BRIEFING-Policy-decisions-on-aspects-of-the-COVID-19-Leave-Support-Scheme.pdf', '1', '85da5a1f35c3b33fd81c36c6c6fd0419');\n",
            "      \n",
            "Saved file locally as: /content/drive/My Drive/downloads2/wage_subsidy_and_leave_schemes/BRIEFING-Policy-decisions-on-aspects-of-the-COVID-19-Leave-Support-Scheme.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release-2020-june/MINUTE-Broadening-Access-to-the-COVID-19-Leave-Payment-Scheme.pdf\">Broadening Access to the COVID-19 Leave Payment Scheme (CBC-20-MIN-0047)<span class=\"fileExt\"> [PDF, 735 KB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>28 April 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Minute</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/MINUTE-Broadening-Access-to-the-COVID-19-Leave-Payment-Scheme.pdf\n",
            "MD5 of File on the Internet is:  5b2cf44d743caa8ed89923b79a690304\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '5b2cf44d743caa8ed89923b79a690304'\n",
            "    LIMIT 1\n",
            "    \n",
            "None\n",
            "\n",
            "      INSERT INTO covid19docs.files (doc_type_future, downloaded_dt_utc, url_of_download, added_by_user_id, doc_hash)\n",
            "      VALUES ('1', '2020-06-25 21:57:40.427604', 'https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/MINUTE-Broadening-Access-to-the-COVID-19-Leave-Payment-Scheme.pdf', '1', '5b2cf44d743caa8ed89923b79a690304');\n",
            "      \n",
            "Saved file locally as: /content/drive/My Drive/downloads2/wage_subsidy_and_leave_schemes/MINUTE-Broadening-Access-to-the-COVID-19-Leave-Payment-Scheme.pdf\n",
            "Number of TD columns found: 3\n",
            "Processing Column: 1\n",
            "<td><a href=\"/assets/resources/proactive-release-2020-june/BRIEFING-Interaction-between-leave-entitlements-and-COVID-19-Support-Schemes.pdf\">Interaction between leave entitlements and COVID-19 Support Schemes<span class=\"fileExt\"> [PDF, 1.2 MB]</span></a></td>\n",
            "Processing Column: 2\n",
            "<td>1 May 2020</td>\n",
            "Processing Column: 3\n",
            "<td>Briefing</td>\n",
            "Link to Download: https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/BRIEFING-Interaction-between-leave-entitlements-and-COVID-19-Support-Schemes.pdf\n",
            "MD5 of File on the Internet is:  27ed03fa078a2fea48040e2b72bec264\n",
            "\n",
            "    SELECT url_of_download FROM covid19docs.files \n",
            "    WHERE doc_hash = '27ed03fa078a2fea48040e2b72bec264'\n",
            "    LIMIT 1\n",
            "    \n",
            "None\n",
            "\n",
            "      INSERT INTO covid19docs.files (doc_type_future, downloaded_dt_utc, url_of_download, added_by_user_id, doc_hash)\n",
            "      VALUES ('1', '2020-06-25 21:57:41.238425', 'https://uniteforrecovery.govt.nz/assets/resources/proactive-release-2020-june/BRIEFING-Interaction-between-leave-entitlements-and-COVID-19-Support-Schemes.pdf', '1', '27ed03fa078a2fea48040e2b72bec264');\n",
            "      \n",
            "Saved file locally as: /content/drive/My Drive/downloads2/wage_subsidy_and_leave_schemes/BRIEFING-Interaction-between-leave-entitlements-and-COVID-19-Support-Schemes.pdf\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}