webScrapping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import os\n",
    "\n",
    "# URL of the webpage containing the PDF links\n",
    "url = 'https://www.icar-crida.res.in/publications_annualreports.html'\n",
    "filenames=[]\n",
    "\n",
    "def download(pdf_links, download_dir):\n",
    "    if not os.path.exists(download_dir):\n",
    "        os.makedirs(download_dir)\n",
    "    try:\n",
    "        for pdf_link in pdf_links:\n",
    "            # Construct the absolute URL if the link is relative\n",
    "            if not pdf_link.startswith('http'):\n",
    "                pdf_link = \"https://www.icar-crida.res.in\" + pdf_link[1:]\n",
    "            # Send a GET request to download the PDF file\n",
    "            pdf_response = requests.get(pdf_link)\n",
    "            pdf_response.raise_for_status()  # Raise an exception for HTTP errors (e.g., 404)\n",
    "            # Save the PDF file with a filename based on its URL\n",
    "            filename = pdf_link.split('/')[-1]\n",
    "            file_path = os.path.join(download_dir, filename)\n",
    "            with open(file_path, 'wb') as f:\n",
    "                f.write(pdf_response.content)\n",
    "            print(f\"Downloaded {filename}\")\n",
    "            filenames.append(filename)\n",
    "    except requests.exceptions.RequestException as e:\n",
    "        print(f\"Failed to download {pdf_link}: {e}\")\n",
    "\n",
    "def links():\n",
    "    try:\n",
    "        # Send a GET request to the webpage\n",
    "        response = requests.get(url)\n",
    "        response.raise_for_status()  # Raise an exception for HTTP errors (e.g., 404)\n",
    "\n",
    "        # Parse the HTML content of the webpage\n",
    "        soup = BeautifulSoup(response.text, 'html.parser')\n",
    "\n",
    "        tags = soup.find_all('div', class_= 'filter-CRIDA')\n",
    "        pdf_links=[]\n",
    "        for tag in tags:\n",
    "          a = tag.find('a', href=re.compile(r'\\.pdf$'))\n",
    "          pdf_links.append(a['href'])\n",
    "\n",
    "        print(len(pdf_links))\n",
    "\n",
    "        # Loop through each filtered PDF link and download the PDF file\n",
    "        download(pdf_links, \"C:\\\\Users\\\\DELL\\\\OneDrive\\\\Desktop\\\\LLM2\\\\data\")\n",
    "\n",
    "    except requests.exceptions.RequestException as e:\n",
    "        print(f\"Error: {e}\")\n",
    "    return pdf_links\n",
    "\n",
    "pdf_links = links()"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}