split_applicants.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# About\n",
    "\n",
    "*Purpose:* split a concatenated PDF of applicants, as downloaded from DTU, into individual application PDFs. Also creates a table of applicant details in Tab-Separated Format (.tsv).\n",
    "\n",
    "**Usage:** in the last cell, replace \"my_concatenated.pdf\" with the path to your concatenated PDF, then run all cells.\n",
    "\n",
    "*Prerequisites:*\n",
    "* The string \"Latest Submission Medium\" must appear in the first page of each application in your concantenated PDF, and only there.\n",
    "* Linux utilities: pdftotext (available by default in Ubuntu).\n",
    "* Python packages: PyPDF2, pdfrw, tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import starmap\n",
    "import pathlib\n",
    "from functools import partial\n",
    "import sys\n",
    "import os\n",
    "import subprocess\n",
    "\n",
    "from tqdm.notebook import tqdm\n",
    "from PyPDF2 import PdfFileWriter, PdfFileReader\n",
    "from pdfrw import PdfReader, PdfWriter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_num_pages(pdf_path):\n",
    "    return PdfFileReader(pdf_path).getNumPages()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_page_text(pdf_path, page_num):\n",
    "    return page_num, subprocess.check_output((\"pdftotext -f %d -l %d %s -\" % (page_num, page_num, pdf_path)).split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_pages_with_application_start(pdf_path):\n",
    "    def is_new_app_start(args):\n",
    "        return 'Latest Submission Medium' in args[1].decode('utf-8').replace('\\r', '').replace('\\n', '')\n",
    "    \n",
    "    return list(map(lambda args: args[0], \n",
    "               filter(is_new_app_start, \n",
    "                      tqdm(map(partial(get_page_text, pdf_path), range(get_num_pages(pdf_path))), \n",
    "                           desc='Page',\n",
    "                           total=get_num_pages(pdf_path))\n",
    "                     )\n",
    "              )\n",
    "    )[1:]  # Remove 0 from beginning of list (not sure why it is there)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fields_to_extract_for_each_candidate():\n",
    "    return ['Last Name', 'First Name', 'Year of Birth', 'City', 'Citizenship of', 'Have you been previously employed by DTU?']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_one_application(output_dir, pdf_path, page_from, page_to):\n",
    "    def extract(out_path, in_path, page_from, page_to):\n",
    "        pdfwrt = PdfWriter()\n",
    "        pdfrd = PdfReader(in_path)\n",
    "        for page_num in range(page_from, page_to):\n",
    "            pdfwrt.addpage(pdfrd.pages[page_num])\n",
    "        pdfwrt.write(out_path)\n",
    "    \n",
    "    def get_field(field):\n",
    "        for page in range(page_from, page_to):\n",
    "            split_text = get_page_text(pdf_path, page)[1].decode('utf-8').split('\\n')\n",
    "            if field in split_text:\n",
    "                return split_text[split_text.index(field) + 1]\n",
    "    \n",
    "    pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)\n",
    "    extract(\n",
    "        out_path=os.path.join(output_dir, '_'.join(map(get_field, fields_to_extract_for_each_candidate())) + '.pdf'),\n",
    "        in_path=pdf_path, \n",
    "        page_from=page_from - 1, \n",
    "        page_to=page_to - 1)\n",
    "    return '\\t'.join(map(get_field, fields_to_extract_for_each_candidate())) + '\\n'\n",
    "\n",
    "\n",
    "# Example usage\n",
    "# extract_one_application(output_dir='.', pdf_path='./print-4.pdf', page_from=163, page_to=168)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_all_applications(output_dir, pdf_path, summary_tsv_path):\n",
    "    pages = find_pages_with_application_start(pdf_path) + [get_num_pages(pdf_path)]\n",
    "    tasks = [(output_dir, pdf_path, pages[i], pages[i + 1]) for i in range(len(pages) - 1)]\n",
    "    with open(summary_tsv_path, 'w') as summary_f:\n",
    "        summary_f.write('\\t'.join(fields_to_extract_for_each_candidate()) + '\\n')\n",
    "        for candidate_fields in tqdm(starmap(extract_one_application, tasks), desc='Candidate', total=len(tasks)):\n",
    "            summary_f.write(candidate_fields)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_all_applications(\n",
    "    output_dir='./out', \n",
    "    pdf_path='./my_concatenated.pdf', \n",
    "    summary_tsv_path='./summary.tsv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}