7_rag_from_scratch.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "AFXeei9yw6"
      },
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "\n",
        "base_url = 'https://pytorch.org/docs/stable'\n",
        "response = requests.get(base_url)\n",
        "response.status_code"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "\n"
        },
        {
          "output_type": "execute_result",
          "execution_count": 1,
          "data": {
            "text/plain": "200"
          },
          "metadata": {}
        }
      ],
      "execution_count": 1
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "2RJZy9lxfO"
      },
      "source": [
        "soup = BeautifulSoup(response.content, 'html.parser')\n",
        "docs_wrapper = soup.select('div.toctree-wrapper.compound')[-5:]\n",
        "len(docs_wrapper)"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "\n"
        },
        {
          "output_type": "execute_result",
          "execution_count": 2,
          "data": {
            "text/plain": "5"
          },
          "metadata": {}
        }
      ],
      "execution_count": 2
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "cVTVgOAXQh"
      },
      "source": [
        "def recurse_tree(root, count=1):\n",
        "    if root.name == 'li' and root.find('ul') is None:\n",
        "        leaf = root.select('a.reference.internal, a.reference.external')[0]\n",
        "        return {leaf.text: leaf['href']}\n",
        "    text = root.select('a.reference.internal, p.caption span.caption-text')[0].text\n",
        "    return {text: [recurse_tree(i, count+1) for i in root.select(f'li.toctree-l{count}')]}\n",
        "docs = [recurse_tree(i) for i in docs_wrapper]\n",
        "len(docs)"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": "\n"
        },
        {
          "output_type": "execute_result",
          "execution_count": 3,
          "data": {
            "text/plain": "5"
          },
          "metadata": {}
        }
      ],
      "execution_count": 3
    },
    {
      "cell_type": "code",
      "metadata": {
        "jukit_cell_id": "qOLGSrjqGO"
      },
      "source": [],
      "outputs": [],
      "execution_count": null
    }
  ],
  "metadata": {
    "anaconda-cloud": {},
    "kernelspec": {
      "display_name": "python",
      "language": "python",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 4
}