download-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2d6db369",
   "metadata": {},
   "source": [
    "# Download NFIP Claims File\n",
    "Author: Mark Bauer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ef359360",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries\n",
    "import requests\n",
    "from datetime import datetime\n",
    "import duckdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "26f6cdc0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python implementation: CPython\n",
      "Python version       : 3.11.0\n",
      "IPython version      : 8.6.0\n",
      "\n",
      "requests: 2.28.1\n",
      "duckdb  : 1.0.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# reproducibility\n",
    "%reload_ext watermark\n",
    "%watermark -v -p requests,duckdb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b057f8d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The data was retrieved on 2024-10-20.\n"
     ]
    }
   ],
   "source": [
    "# data retrieved\n",
    "current_date = datetime.now()\n",
    "print(f\"The data was retrieved on {current_date.strftime('%Y-%m-%d')}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "46025b2e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File downloaded successfully.\n"
     ]
    }
   ],
   "source": [
    "# URL of the CSV file to download\n",
    "url = \"https://www.fema.gov/about/reports-and-data/openfema/FimaNfipClaims.csv\"\n",
    "\n",
    "# Send an HTTP GET request to the URL\n",
    "response = requests.get(url)\n",
    "\n",
    "# Check if the request was successful (status code 200)\n",
    "if response.status_code == 200:\n",
    "    \n",
    "    # Specify the file path where you want to save the downloaded file\n",
    "    file_path = \"data/FimaNfipClaims.csv\"\n",
    "    \n",
    "    # Write the content of the response to a file\n",
    "    with open(file_path, 'wb') as file:\n",
    "        file.write(response.content)\n",
    "    print(\"File downloaded successfully.\")\n",
    "    \n",
    "else:\n",
    "    print(\"Failed to download file.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "081f36fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FimaNfipClaims.csv\r\n"
     ]
    }
   ],
   "source": [
    "# confirm file in data/ folder\n",
    "%ls data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4625055e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "985M\tdata\r\n",
      "288K\tanalysis.ipynb\r\n",
      "264K\tfigures\r\n",
      "168K\tmap.ipynb\r\n",
      "8.0K\tdownload-data.ipynb\r\n",
      "4.0K\tREADME.md\r\n",
      "4.0K\tLICENSE\r\n"
     ]
    }
   ],
   "source": [
    "# list file sizes\n",
    "!du -sh * | sort -rh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "18289792",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "985M\tdata/FimaNfipClaims.csv\r\n"
     ]
    }
   ],
   "source": [
    "# preview size of file\n",
    "!du -sh data/* | sort -rh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "96bf0b49",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "┌──────────────────────┬──────────────────────┬──────────────────────┬───┬──────────┬───────────┬──────────────────────┐\n",
       "│ agricultureStructu…  │       asOfDate       │ basementEnclosureC…  │ … │ latitude │ longitude │          id          │\n",
       "│        int64         │      timestamp       │        int64         │   │  double  │  double   │       varchar        │\n",
       "├──────────────────────┼──────────────────────┼──────────────────────┼───┼──────────┼───────────┼──────────────────────┤\n",
       "│                    0 │ 2020-11-13 14:50:3…  │                    2 │ … │     30.3 │     -87.7 │ e7af3d9f-b605-4653…  │\n",
       "│                    0 │ 2020-12-11 16:25:4…  │                    0 │ … │     30.3 │     -87.7 │ bbaeaf64-c162-41bf…  │\n",
       "│                    0 │ 2020-03-27 12:15:4…  │                    0 │ … │     33.5 │    -112.1 │ 256da746-b30b-4129…  │\n",
       "│                    0 │ 2020-03-27 12:15:4…  │                    0 │ … │     33.5 │    -112.1 │ e3dcbb27-a2d0-4a9e…  │\n",
       "│                    0 │ 2020-03-26 12:56:2…  │                    0 │ … │     34.4 │    -119.6 │ f77efb94-0188-4fa0…  │\n",
       "│                    0 │ 2020-02-28 12:34:1…  │                    0 │ … │     41.1 │     -73.4 │ 4119d7f2-4df9-4ee5…  │\n",
       "│                    0 │ 2020-01-22 16:55:5…  │                    0 │ … │     25.9 │     -81.7 │ 623ad8d6-5ce8-4560…  │\n",
       "│                    0 │ 2022-07-28 11:35:3…  │                    0 │ … │     30.4 │     -86.9 │ a004f4fc-1435-4641…  │\n",
       "│                    0 │ 2022-07-28 11:35:3…  │                    0 │ … │     30.4 │     -86.9 │ 20684013-1ab2-4acc…  │\n",
       "│                    0 │ 2023-03-31 20:23:4…  │                    0 │ … │     25.6 │     -80.5 │ e9c126c0-37a3-42bb…  │\n",
       "├──────────────────────┴──────────────────────┴──────────────────────┴───┴──────────┴───────────┴──────────────────────┤\n",
       "│ 10 rows                                                                                         73 columns (6 shown) │\n",
       "└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sanity check on exported file\n",
    "duckdb.sql(\"SELECT * FROM read_csv('data/FimaNfipClaims.csv') LIMIT 10;\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65dfef01",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}