langchain_gemini_rag_github.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "USER_AGENT environment variable not set, consider setting it to identify your requests.\n"
     ]
    }
   ],
   "source": [
    "from langchain.document_loaders import WebBaseLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = WebBaseLoader(\"https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md\")\n",
    "pages = html.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nanishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nSkip to content\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nNavigation Menu\\n\\nToggle navigation\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n            Sign in\\n          \\n\\n\\n\\n\\n\\n\\n\\n\\n        Product\\n        \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nActions\\n        Automate any workflow\\n      \\n\\n\\n\\n\\n\\n\\n\\nPackages\\n        Host and manage packages\\n      \\n\\n\\n\\n\\n\\n\\n\\nSecurity\\n        Find and fix vulnerabilities\\n      \\n\\n\\n\\n\\n\\n\\n\\nCodespaces\\n        Instant dev environments\\n      \\n\\n\\n\\n\\n\\n\\n\\nGitHub Copilot\\n        Write better code with AI\\n      \\n\\n\\n\\n\\n\\n\\n\\nCode review\\n        Manage code changes\\n      \\n\\n\\n\\n\\n\\n\\n\\nIssues\\n        Plan and track work\\n      \\n\\n\\n\\n\\n\\n\\n\\nDiscussions\\n        Collaborate outside of code\\n      \\n\\n\\n\\n\\nExplore\\n\\n\\n\\n      All features\\n\\n    \\n\\n\\n\\n      Documentation\\n\\n    \\n\\n\\n\\n\\n\\n      GitHub Skills\\n\\n    \\n\\n\\n\\n\\n\\n      Blog\\n\\n    \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        Solutions\\n        \\n\\n\\n\\n\\n\\nBy size\\n\\n\\n\\n      Enterprise\\n\\n    \\n\\n\\n\\n      Teams\\n\\n    \\n\\n\\n\\n      Startups\\n\\n    \\n\\n\\n\\n\\nBy industry\\n\\n\\n\\n      Healthcare\\n\\n    \\n\\n\\n\\n      Financial services\\n\\n    \\n\\n\\n\\n      Manufacturing\\n\\n    \\n\\n\\n\\n\\nBy use case\\n\\n\\n\\n      CI/CD & Automation\\n\\n    \\n\\n\\n\\n      DevOps\\n\\n    \\n\\n\\n\\n      DevSecOps\\n\\n    \\n\\n\\n\\n\\n\\n\\n\\n        Resources\\n        \\n\\n\\n\\n\\n\\nTopics\\n\\n\\n\\n      AI\\n\\n    \\n\\n\\n\\n      DevOps\\n\\n    \\n\\n\\n\\n      Innersource\\n\\n    \\n\\n\\n\\n      Open Source\\n\\n    \\n\\n\\n\\n      Security\\n\\n    \\n\\n\\n\\n      Software Development\\n\\n    \\n\\n\\n\\n\\nExplore\\n\\n\\n\\n      Learning Pathways\\n\\n    \\n\\n\\n\\n\\n\\n      White papers, Ebooks, Webinars\\n\\n    \\n\\n\\n\\n\\n\\n      Customer Stories\\n\\n    \\n\\n\\n\\n      Partners\\n\\n    \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        Open Source\\n        \\n\\n\\n\\n\\n\\n\\n\\n\\n\\nGitHub Sponsors\\n        Fund open source developers\\n      \\n\\n\\n\\n\\n\\n\\n\\n\\nThe ReadME Project\\n        GitHub community articles\\n      \\n\\n\\n\\n\\nRepositories\\n\\n\\n\\n      Topics\\n\\n    \\n\\n\\n\\n      Trending\\n\\n    \\n\\n\\n\\n      Collections\\n\\n    \\n\\n\\n\\n\\n\\n\\n\\n        Enterprise\\n        \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nEnterprise platform\\n        AI-powered developer platform\\n      \\n\\n\\n\\n\\nAvailable add-ons\\n\\n\\n\\n\\n\\n\\n\\nAdvanced Security\\n        Enterprise-grade security features\\n      \\n\\n\\n\\n\\n\\n\\n\\nGitHub Copilot\\n        Enterprise-grade AI features\\n      \\n\\n\\n\\n\\n\\n\\n\\nPremium Support\\n        Enterprise-grade 24/7 support\\n      \\n\\n\\n\\n\\n\\n\\nPricing\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nSearch or jump to...\\n\\n\\n\\n\\n\\n\\n\\nSearch code, repositories, users, issues, pull requests...\\n\\n \\n\\n\\n\\n\\n        Search\\n      \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nClear\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n              Search syntax tips\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        Provide feedback\\n      \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\nWe read every piece of feedback, and take your input very seriously.\\n\\n\\nInclude my email address so I can be contacted\\n\\n\\n     Cancel\\n\\n    Submit feedback\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        Saved searches\\n      \\nUse saved searches to filter your results more quickly\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\nName\\n\\n\\n\\n\\n\\n\\nQuery\\n\\n\\n\\n            To see all available qualifiers, see our documentation.\\n          \\n \\n\\n\\n\\n\\n\\n     Cancel\\n\\n    Create saved search\\n\\n\\n\\n\\n\\n\\n\\n\\n                Sign in\\n              \\n\\n\\n                Sign up\\n              \\n\\n\\n\\n\\n\\n\\n\\n\\n\\nYou signed in with another tab or window. Reload to refresh your session.\\nYou signed out in another tab or window. Reload to refresh your session.\\nYou switched accounts on another tab or window. Reload to refresh your session.\\n \\n\\n\\nDismiss alert\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        anishchapagain\\n \\n/\\n\\nanishchapagain.github.io\\n\\nPublic\\n\\n\\n\\n\\n\\n \\n\\nNotifications\\n You must be signed in to change notification settings\\n\\n\\n \\n\\nFork\\n    0\\n\\n\\n\\n\\n \\n\\n\\n          Star\\n 0\\n  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nCode\\n\\n\\n\\n\\n\\n\\n\\nIssues\\n0\\n\\n\\n\\n\\n\\n\\nPull requests\\n0\\n\\n\\n\\n\\n\\n\\nActions\\n\\n\\n\\n\\n\\n\\n\\nProjects\\n0\\n\\n\\n\\n\\n\\n\\nSecurity\\n\\n\\n\\n\\n\\n\\n\\nInsights\\n\\n\\n\\n \\n\\n \\n\\n\\nAdditional navigation options\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Code\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Issues\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Pull requests\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Actions\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Projects\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Security\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Insights\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nFooter\\n\\n\\n\\n\\n\\n\\n\\n\\n        © 2024 GitHub,\\xa0Inc.\\n      \\n\\n\\nFooter navigation\\n\\n\\nTerms\\n\\n\\nPrivacy\\n\\n\\nSecurity\\n\\n\\nStatus\\n\\n\\nDocs\\n\\n\\nContact\\n\\n\\n\\n\\n      Manage cookies\\n    \\n\\n\\n\\n\\n\\n      Do not share my personal information\\n    \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n    You can’t perform that action at this time.\\n  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'})]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import RecursiveCharacterTextSplitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "separators=[\n",
    "        \"\\n\\n\",\n",
    "        \"\\n\",\n",
    "        \" \",\n",
    "        \".\",\n",
    "        \",\",\n",
    "        \"\\u200b\",  # Zero-width space\n",
    "        \"\\uff0c\",  # Fullwidth comma\n",
    "        \"\\u3001\",  # Ideographic comma\n",
    "        \"\\uff0e\",  # Fullwidth full stop\n",
    "        \"\\u3002\",  # Ideographic full stop\n",
    "        \"\",\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "r_splitter = RecursiveCharacterTextSplitter(\n",
    "    chunk_size=100,\n",
    "    chunk_overlap=5,\n",
    "    separators=separators\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = r_splitter.split_documents(pages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Skip to content\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nNavigation Menu', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Toggle navigation\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n            Sign in', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Product\\n        \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nActions\\n        Automate any workflow', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Packages\\n        Host and manage packages', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Security\\n        Find and fix vulnerabilities', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Codespaces\\n        Instant dev environments', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='GitHub Copilot\\n        Write better code with AI', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Code review\\n        Manage code changes', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Issues\\n        Plan and track work', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Discussions\\n        Collaborate outside of code\\n      \\n\\n\\n\\n\\nExplore\\n\\n\\n\\n      All features', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Documentation\\n\\n    \\n\\n\\n\\n\\n\\n      GitHub Skills\\n\\n    \\n\\n\\n\\n\\n\\n      Blog', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Solutions\\n        \\n\\n\\n\\n\\n\\nBy size\\n\\n\\n\\n      Enterprise\\n\\n    \\n\\n\\n\\n      Teams', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Startups\\n\\n    \\n\\n\\n\\n\\nBy industry\\n\\n\\n\\n      Healthcare\\n\\n    \\n\\n\\n\\n      Financial services', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Manufacturing\\n\\n    \\n\\n\\n\\n\\nBy use case\\n\\n\\n\\n      CI/CD & Automation\\n\\n    \\n\\n\\n\\n      DevOps', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='DevSecOps\\n\\n    \\n\\n\\n\\n\\n\\n\\n\\n        Resources\\n        \\n\\n\\n\\n\\n\\nTopics\\n\\n\\n\\n      AI', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='DevOps\\n\\n    \\n\\n\\n\\n      Innersource\\n\\n    \\n\\n\\n\\n      Open Source\\n\\n    \\n\\n\\n\\n      Security', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Software Development\\n\\n    \\n\\n\\n\\n\\nExplore\\n\\n\\n\\n      Learning Pathways', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='White papers, Ebooks, Webinars\\n\\n    \\n\\n\\n\\n\\n\\n      Customer Stories\\n\\n    \\n\\n\\n\\n      Partners', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Open Source', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='GitHub Sponsors\\n        Fund open source developers', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='The ReadME Project\\n        GitHub community articles\\n      \\n\\n\\n\\n\\nRepositories\\n\\n\\n\\n      Topics', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Trending\\n\\n    \\n\\n\\n\\n      Collections\\n\\n    \\n\\n\\n\\n\\n\\n\\n\\n        Enterprise', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Enterprise platform\\n        AI-powered developer platform\\n      \\n\\n\\n\\n\\nAvailable add-ons', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Advanced Security\\n        Enterprise-grade security features', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='GitHub Copilot\\n        Enterprise-grade AI features', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Premium Support\\n        Enterprise-grade 24/7 support\\n      \\n\\n\\n\\n\\n\\n\\nPricing', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Search or jump to...\\n\\n\\n\\n\\n\\n\\n\\nSearch code, repositories, users, issues, pull requests...', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Search\\n      \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nClear', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Search syntax tips\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        Provide feedback', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='We read every piece of feedback, and take your input very seriously.', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Include my email address so I can be contacted\\n\\n\\n     Cancel\\n\\n    Submit feedback', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Saved searches\\n      \\nUse saved searches to filter your results more quickly', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Name\\n\\n\\n\\n\\n\\n\\nQuery', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='To see all available qualifiers, see our documentation.', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Cancel\\n\\n    Create saved search\\n\\n\\n\\n\\n\\n\\n\\n\\n                Sign in', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Sign up', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='You signed in with another tab or window. Reload to refresh your session.', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='You signed out in another tab or window. Reload to refresh your session.', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='You switched accounts on another tab or window. Reload to refresh your session.', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Dismiss alert\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        anishchapagain\\n \\n/\\n\\nanishchapagain.github.io\\n\\nPublic', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Notifications\\n You must be signed in to change notification settings\\n\\n\\n \\n\\nFork\\n    0', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Star\\n 0\\n  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nCode\\n\\n\\n\\n\\n\\n\\n\\nIssues\\n0\\n\\n\\n\\n\\n\\n\\nPull requests\\n0\\n\\n\\n\\n\\n\\n\\nActions', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Projects\\n0\\n\\n\\n\\n\\n\\n\\nSecurity\\n\\n\\n\\n\\n\\n\\n\\nInsights\\n\\n\\n\\n \\n\\n \\n\\n\\nAdditional navigation options', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Code\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Issues\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Pull requests', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Actions\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Projects\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Security', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Insights\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nFooter\\n\\n\\n\\n\\n\\n\\n\\n\\n        © 2024 GitHub,\\xa0Inc.', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Footer navigation\\n\\n\\nTerms\\n\\n\\nPrivacy\\n\\n\\nSecurity\\n\\n\\nStatus\\n\\n\\nDocs\\n\\n\\nContact', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='Manage cookies\\n    \\n\\n\\n\\n\\n\\n      Do not share my personal information', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'}),\n",
       " Document(page_content='You can’t perform that action at this time.', metadata={'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub', 'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en'})]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs[0].page_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read API KEY from file\n",
    "import json\n",
    "\n",
    "file_path = 'llms_vault.json'\n",
    "llm_key = 'GOOGLE_API_KEY'\n",
    "\n",
    "try:\n",
    "    with open(file_path,'r') as f:\n",
    "        api_key = json.loads(f.read()).get(llm_key)\n",
    "        if not api_key:\n",
    "            raise ValueError(f\"Some issue with '{llm_key}' from '{file_path}'. Verify the key:'{llm_key}' if it exists!\")\n",
    "except FileNotFoundError:\n",
    "    raise FileNotFoundError(f\"File related issue encountered, verify file path and name '{file_path}'\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# https://ai.google.dev/api\n",
    "import google.generativeai as genai\n",
    "genai.configure(api_key=api_key) # genai.configure(api_key=os.environ[llm_key]) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_google_genai import GoogleGenerativeAIEmbeddings # models/text-embedding-004\t"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "import os\n",
    "os.environ[\"GOOGLE_API_KEY\"] = api_key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#result = genai.embed_content(model=\"models/text-embedding-004\", content=docsr)\n",
    "\n",
    "embedding_model = GoogleGenerativeAIEmbeddings(\n",
    "    model=\"models/embedding-001\", # text-embedding-004\n",
    "    google_api_key=api_key,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GoogleGenerativeAIEmbeddings(client=<google.ai.generativelanguage_v1beta.services.generative_service.client.GenerativeServiceClient object at 0x000001BE40BCAFC0>, model='models/embedding-001', task_type=None, google_api_key=SecretStr('**********'), credentials=None, client_options=None, transport=None, request_options=None)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.vectorstores import Chroma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "persist_directory = 'docs\\\\chromaE'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectordb = Chroma.from_documents(\n",
    "    documents=docs,\n",
    "    embedding=embedding_model,\n",
    "    persist_directory=persist_directory,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "vectordb = Chroma.from_documents(\n",
    "    documents=docsr,\n",
    "    embedding=embeddings,\n",
    "    persist_directory=persist_directory\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<langchain_community.vectorstores.chroma.Chroma at 0x1be40d5cb30>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectordb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "50\n"
     ]
    }
   ],
   "source": [
    "print(vectordb._collection.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vector: 50 chunks.\n"
     ]
    }
   ],
   "source": [
    "googleDb = Chroma(\n",
    "    persist_directory = persist_directory,\n",
    "    embedding_function=embedding_model\n",
    "    )\n",
    "print(\"vector:\",googleDb._collection.count(),\"chunks.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<langchain_community.vectorstores.chroma.Chroma at 0x1be40d5c680>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "googleDb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "retriver = googleDb.as_retriever() \n",
    "# search_type=\"similarity\",k=4)\n",
    "# search_type=\"mmr\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Search or jump to...\\n\\n\\n\\n\\n\\n\\n\\nSearch code, repositories, users, issues, pull requests...', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'}),\n",
       " Document(page_content='Dismiss alert\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        anishchapagain\\n \\n/\\n\\nanishchapagain.github.io\\n\\nPublic', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'}),\n",
       " Document(page_content='The ReadME Project\\n        GitHub community articles\\n      \\n\\n\\n\\n\\nRepositories\\n\\n\\n\\n      Topics', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'})]"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs_mmr = googleDb.max_marginal_relevance_search(\"list available URLs with text as provided in triple ticks '''github.io'''\",k=3)\n",
    "docs_mmr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VectorStoreRetriever(tags=['Chroma', 'GoogleGenerativeAIEmbeddings'], vectorstore=<langchain_community.vectorstores.chroma.Chroma object at 0x000001BE40D5C680>)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retriver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Search or jump to...\\n\\n\\n\\n\\n\\n\\n\\nSearch code, repositories, users, issues, pull requests...', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'}),\n",
       " Document(page_content='Insights\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nFooter\\n\\n\\n\\n\\n\\n\\n\\n\\n        © 2024 GitHub,\\xa0Inc.', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'}),\n",
       " Document(page_content='Dismiss alert\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        anishchapagain\\n \\n/\\n\\nanishchapagain.github.io\\n\\nPublic', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'}),\n",
       " Document(page_content='Star\\n 0\\n  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nCode\\n\\n\\n\\n\\n\\n\\n\\nIssues\\n0\\n\\n\\n\\n\\n\\n\\nPull requests\\n0\\n\\n\\n\\n\\n\\n\\nActions', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'})]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = retriver.get_relevant_documents(\"list available URLs with text as provided in triple ticks '''github.io'''\")\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = retriver.invoke(\"list available URLs with text as provided in triple ticks '''github.io'''\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    }
   ],
   "source": [
    "print(len(llm))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Document(page_content='Search or jump to...\\n\\n\\n\\n\\n\\n\\n\\nSearch code, repositories, users, issues, pull requests...', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'}),\n",
       " Document(page_content='Insights\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nFooter\\n\\n\\n\\n\\n\\n\\n\\n\\n        © 2024 GitHub,\\xa0Inc.', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'}),\n",
       " Document(page_content='Dismiss alert\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        anishchapagain\\n \\n/\\n\\nanishchapagain.github.io\\n\\nPublic', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'}),\n",
       " Document(page_content='Star\\n 0\\n  \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nCode\\n\\n\\n\\n\\n\\n\\n\\nIssues\\n0\\n\\n\\n\\n\\n\\n\\nPull requests\\n0\\n\\n\\n\\n\\n\\n\\nActions', metadata={'description': 'Anish Chapagain. Contribute to anishchapagain/anishchapagain.github.io development by creating an account on GitHub.', 'language': 'en', 'source': 'https://github.com/anishchapagain/anishchapagain.github.io/blob/master/README.md', 'title': 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub'})]"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_google_genai import ChatGoogleGenerativeAI\n",
    "from langchain.schema import StrOutputParser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_model = ChatGoogleGenerativeAI(\n",
    "    model=\"gemini-1.5-flash-latest\",\n",
    "    google_api_key=api_key,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain import PromptTemplate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "llm_prompt_template = \"\"\"You are an assistant for question-answering tasks. Use the following context to answer the question.\n",
    "If you don't know the answer, just say that you don't know. Use two to three sentences maximum and keep the answer concise.\\n\n",
    "Question: {question} \\nContext: {context} \\nAnswer:\"\"\"\n",
    "\n",
    "llm_prompt_template = \"\"\"You are an assistant, use the following context to answer the question.\n",
    "Use two to three sentences maximum and keep the answer concise.\\n\n",
    "Question: {question} \\nContext: {context} \\nAnswer:\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_variables=['context', 'question'] template='You are an assistant, use the following context to answer the question.\\nUse two to three sentences maximum and keep the answer concise.\\n\\nQuestion: {question} \\nContext: {context} \\nAnswer:'\n"
     ]
    }
   ],
   "source": [
    "llm_prompt = PromptTemplate.from_template(llm_prompt_template)\n",
    "print(llm_prompt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def format_docs(docs):\n",
    "    return \"\\n\\n\".join(doc.page_content for doc in docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.schema.runnable import RunnablePassthrough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reply(question=''):\n",
    "    chain = (\n",
    "    {\"context\": retriver | format_docs, \"question\": RunnablePassthrough()}\n",
    "    | llm_prompt | llm_model | StrOutputParser()\n",
    "    )\n",
    "    print(chain.invoke(question))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I'm sorry, but the provided context \"Cancel\" does not contain any information about cities or countries. Please provide a different context. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply(\"List name of some cities and country\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The provided context does not include information about the location of Himalayan Whitehouse College. Therefore, I cannot answer your question. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply(\"where is Himalayan Whitehouse College located\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are likely logged into the same account on multiple tabs or windows. To resolve this, refresh the current tab to sync your session and ensure you are using the correct account. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This document is a GitHub repository landing page for the \"ReadME Project,\" which appears to focus on community articles. It includes sections for customer stories, partners, repositories, topics, and footer navigation. The page also provides links to terms, privacy, security, status, docs, and contact information. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply(\"summarize the whole document in less that 200 words\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The provided context mentions roles related to CI/CD & Automation, DevOps, and Security.  These roles likely include engineers, developers, and security analysts. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply(\"provide me some technical roles mentioned\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The provided context doesn't contain any text with the phrase \"github.io.\" The only instance of \"github.io\" is in the URL \"anishchapagain.github.io/README.md.\" \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply(\"provide me full texts containing the word 'github.io'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You can contribute by submitting feedback, opening issues, or creating pull requests.  The website encourages users to provide input and engage with the project. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply(\"can you list me few places as where can i contribute to\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The content emphasizes that projects are inactive and lack purpose without actions.  They require concrete steps to be implemented and brought to life. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply(\"what the content is tyring to say as 'Projects are life-less without'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I cannot provide you with a quote from Sadhguru. I do not have access to the internet or external websites, including the one you provided. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply(\"some quote by sadhguru is there, provide me the quote\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The author is willing to be trained on DevSecOps, AI, code, issues, and pull requests. These topics are listed under the \"Resources\" section of the documentation. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply(\"what are the topics author is willing to get trained on\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The provided context doesn't contain any URLs, links, or email addresses. It only shows a footer navigation with terms like Privacy, Security, Status, Docs, and Contact. \n",
      "\n"
     ]
    }
   ],
   "source": [
    "reply(\"list me URLs or links or email address from the page\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search(question='') -> list:\n",
    "    docs = googleDb.similarity_search(question, k=3)\n",
    "    answers=[]\n",
    "    for doc in docs:\n",
    "        answers.append(doc.page_content[:200])\n",
    "    \n",
    "    return answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Code\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Issues\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Pull requests',\n",
       " 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub',\n",
       " 'We read every piece of feedback, and take your input very seriously.']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search(question=\"list me some places where can i contribute to\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Search or jump to...\\n\\n\\n\\n\\n\\n\\n\\nSearch code, repositories, users, issues, pull requests...',\n",
       " 'Insights\\n\\n\\n\\n\\n\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\nFooter\\n\\n\\n\\n\\n\\n\\n\\n\\n        © 2024 GitHub,\\xa0Inc.',\n",
       " 'Dismiss alert\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        anishchapagain\\n \\n/\\n\\nanishchapagain.github.io\\n\\nPublic']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search(question=\"list available URLs with text as provided in triple ticks '''github.io'''\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Notifications\\n You must be signed in to change notification settings\\n\\n\\n \\n\\nFork\\n    0',\n",
       " 'Documentation\\n\\n    \\n\\n\\n\\n\\n\\n      GitHub Skills\\n\\n    \\n\\n\\n\\n\\n\\n      Blog',\n",
       " 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub']"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search(question=\"what is there mentioned as Willing to get trained in\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Discussions\\n        Collaborate outside of code\\n      \\n\\n\\n\\n\\nExplore\\n\\n\\n\\n      All features',\n",
       " 'Issues\\n        Plan and track work',\n",
       " 'Code\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Issues\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n          Pull requests']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search(\"what we can work together in any topics\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Search syntax tips\\n \\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n        Provide feedback',\n",
       " 'Footer navigation\\n\\n\\nTerms\\n\\n\\nPrivacy\\n\\n\\nSecurity\\n\\n\\nStatus\\n\\n\\nDocs\\n\\n\\nContact',\n",
       " 'anishchapagain.github.io/README.md at master · anishchapagain/anishchapagain.github.io · GitHub']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "search(\"list any text that's similar to honesty\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}