04_loadsave.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to DataFrames\n",
    "**[Bogumił Kamiński](http://bogumilkaminski.pl/about/), Apr 21, 2018**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "using DataFrames # load package"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and save DataFrames\n",
    "We do not cover all features of the packages. Please refer to their documentation to learn them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "using CSV # reading and writing CSV files\n",
    "using JLD # Julia native binary format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>A</th><th>B</th><th>C</th><th>D</th></tr></thead><tbody><tr><th>1</th><td>true</td><td>1</td><td>missing</td><td>'a'</td></tr><tr><th>2</th><td>false</td><td>2</td><td>b</td><td>missing</td></tr><tr><th>3</th><td>true</td><td>missing</td><td>c</td><td>'c'</td></tr></tbody></table>"
      ],
      "text/plain": [
       "3×4 DataFrames.DataFrame\n",
       "│ Row │ A     │ B       │ C       │ D       │\n",
       "├─────┼───────┼─────────┼─────────┼─────────┤\n",
       "│ 1   │ true  │ 1       │ \u001b[90mmissing\u001b[39m │ 'a'     │\n",
       "│ 2   │ false │ 2       │ b       │ \u001b[90mmissing\u001b[39m │\n",
       "│ 3   │ true  │ \u001b[90mmissing\u001b[39m │ c       │ 'c'     │"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = DataFrame(A=[true, false, true], B=[1, 2, missing],\n",
    "              C=[missing, \"b\", \"c\"], D=['a', missing, 'c']) # create a simple DataFrame for testing purposes\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element Array{Type,1}:\n",
       " Bool                           \n",
       " Union{Int64, Missings.Missing} \n",
       " Union{Missings.Missing, String}\n",
       " Union{Char, Missings.Missing}  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eltypes(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CSV.Sink{Void,DataType}(    CSV.Options:\n",
       "        delim: ','\n",
       "        quotechar: '\"'\n",
       "        escapechar: '\\\\'\n",
       "        missingstring: \"\"\n",
       "        dateformat: nothing\n",
       "        decimal: '.'\n",
       "        truestring: 'true'\n",
       "        falsestring: 'false', IOBuffer(data=UInt8[...], readable=true, writable=true, seekable=true, append=false, size=0, maxsize=Inf, ptr=1, mark=-1), \"x.csv\", 8, true, String[\"A\", \"B\", \"C\", \"D\"], 4, false, Val{false})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "CSV.write(\"x.csv\", x) # save it to disk; make sure x.csv does not conflict with some file in your working directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A,B,C,D\n",
      "true,1,,a\n",
      "false,2,b,\n",
      "true,,c,c\n"
     ]
    }
   ],
   "source": [
    "print(read(\"x.csv\", String)) # this is how it was saved"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>A</th><th>B</th><th>C</th><th>D</th></tr></thead><tbody><tr><th>1</th><td>true</td><td>1</td><td>missing</td><td>a</td></tr><tr><th>2</th><td>false</td><td>2</td><td>b</td><td>missing</td></tr><tr><th>3</th><td>true</td><td>missing</td><td>c</td><td>c</td></tr></tbody></table>"
      ],
      "text/plain": [
       "3×4 DataFrames.DataFrame\n",
       "│ Row │ A     │ B       │ C       │ D       │\n",
       "├─────┼───────┼─────────┼─────────┼─────────┤\n",
       "│ 1   │ true  │ 1       │ \u001b[90mmissing\u001b[39m │ a       │\n",
       "│ 2   │ false │ 2       │ b       │ \u001b[90mmissing\u001b[39m │\n",
       "│ 3   │ true  │ \u001b[90mmissing\u001b[39m │ c       │ c       │"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we can load it back; disable memory mapping so that on Windows the file can be deleted in the same session\n",
    "y = CSV.read(\"x.csv\", use_mmap=false)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element Array{Type,1}:\n",
       " Union{Bool, Missings.Missing}  \n",
       " Union{Int64, Missings.Missing} \n",
       " Union{Missings.Missing, String}\n",
       " Union{Missings.Missing, String}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eltypes(y) # all columns allow Missing by default; also column types have changed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "save(\"x.jld\", \"x\", x) # save to a file in a binary format; make sure that x.jld does not exist in your working directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table class=\"data-frame\"><thead><tr><th></th><th>A</th><th>B</th><th>C</th><th>D</th></tr></thead><tbody><tr><th>1</th><td>true</td><td>1</td><td>missing</td><td>'a'</td></tr><tr><th>2</th><td>false</td><td>2</td><td>b</td><td>missing</td></tr><tr><th>3</th><td>true</td><td>missing</td><td>c</td><td>'c'</td></tr></tbody></table>"
      ],
      "text/plain": [
       "3×4 DataFrames.DataFrame\n",
       "│ Row │ A     │ B       │ C       │ D       │\n",
       "├─────┼───────┼─────────┼─────────┼─────────┤\n",
       "│ 1   │ true  │ 1       │ \u001b[90mmissing\u001b[39m │ 'a'     │\n",
       "│ 2   │ false │ 2       │ b       │ \u001b[90mmissing\u001b[39m │\n",
       "│ 3   │ true  │ \u001b[90mmissing\u001b[39m │ c       │ 'c'     │"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = load(\"x.jld\", \"x\") # this is identical to x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4-element Array{Type,1}:\n",
       " Bool                           \n",
       " Union{Int64, Missings.Missing} \n",
       " Union{Missings.Missing, String}\n",
       " Union{Char, Missings.Missing}  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eltypes(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0.782157 seconds (688.90 k allocations: 30.828 MiB, 1.08% gc time)\n",
      "  0.018250 seconds (203.61 k allocations: 3.339 MiB)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2-element Array{Int64,1}:\n",
       " 595307\n",
       " 154487"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# again bigdf.csv and bigdf.jld files will be created so be careful\n",
    "bigdf = DataFrame(Bool, 10^3, 10^2) # 10^3 rows, 10^5 columns\n",
    "@time CSV.write(\"bigdf.csv\", bigdf)\n",
    "@time save(\"bigdf.jld\", \"bigdf\", bigdf)\n",
    "getfield.(stat.([\"bigdf.csv\", \"bigdf.jld\"]), :size) #  you can expect JLD to be faster, use compress=true to reduce file size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clean up - do not run unless you are sure that it will not erase your important files\n",
    "foreach(rm, [\"x.csv\", \"x.jld\", \"bigdf.csv\", \"bigdf.jld\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.6.2",
   "language": "julia",
   "name": "julia-0.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}