Skip to content

Commit

Permalink
Merge pull request TileDB-Inc#30 from TileDB-Inc/nb/exactearth
Browse files Browse the repository at this point in the history 
exactEarth sample notebooks
  • Loading branch information
@Shelnutt2
Shelnutt2 authored Jan 15, 2021
2 parents f977351 + ce0ed23 commit 843eb65
Show file tree
Hide file tree
Showing 4 changed files with 567 additions and 0 deletions.
124 changes: 124 additions & 0 deletions geo/exactEarth/Ship_Tracks.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Track demo\n",
"\n",
"Track vessel by mmsi"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"\n",
"import datashader as ds\n",
"from datashader.utils import lnglat_to_meters\n",
"import holoviews as hv\n",
"import holoviews.operation.datashader as hd\n",
"from holoviews.element import tiles\n",
"import matplotlib.pyplot as plt\n",
"from matplotlib import cm\n",
"import geopandas as gpd\n",
"import pandas as pd\n",
"import tiledb\n",
"\n",
"hv.extension(\"bokeh\", \"matplotlib\")\n",
"\n",
"tiledb_mmsi_uri = 'tiledb://exactEarth/ais_aug_2019_mmsi'"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The mmsi identifier we will use is `308371000`"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"\n",
"with tiledb.SparseArray(tiledb_mmsi_uri) as arr:\n",
" pts = arr[308371000]\n",
"\n",
"print(f\"Retrieved {len(pts['longitude'])} vessels\")\n",
" \n",
"df = pd.DataFrame(pts)\n",
"df.sort_values(by=['ts_pos_utc'], inplace=True, ascending=True)\n",
"df.loc[:, 'x'], df.loc[:, 'y'] = lnglat_to_meters(df.longitude,df.latitude)\n",
"df"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Using existing data science tooling we can plot these points on a map"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"gdf = gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.longitude, df.latitude), crs='epsg:4326')\n",
"ax = gdf.plot(markersize=1.5, figsize=(8, 8))\n",
"plt.autoscale(False)\n",
"world = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres'))\n",
"world.to_crs(gdf.crs).plot(ax=ax, color='none', edgecolor='black')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"\n",
"bkgrd = tiles.EsriImagery().opts(xaxis=None, yaxis=None, width=700, height=500)\n",
"opts = hv.opts.RGB(width=500, height=500)\n",
"bkgrd * hd.datashade(hv.Path(gdf, kdims=['x','y']), cmap='red', normalization='linear', aggregator=ds.any()).opts(opts)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.8"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
126 changes: 126 additions & 0 deletions geo/exactEarth/Ships_DensityMapShader.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Ships demo\n",
"\n",
"Load and visualize vessel locations from the month of August 2019"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"\n",
"from datashader.utils import lnglat_to_meters\n",
"import holoviews as hv\n",
"import holoviews.operation.datashader as hd\n",
"from holoviews.element import tiles\n",
"from matplotlib import cm\n",
"import pandas as pd\n",
"import tiledb\n",
"\n",
"hv.extension(\"bokeh\", \"matplotlib\")\n",
"\n",
"tiledb_uri = 'tiledb://exactEarth/ais_aug_2019'"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Query for the area around Southampton, England"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"\n",
"x1, x2, y1, y2 = [-1.7, -1.2, 50, 51.0]\n",
"\n",
"st = time.time()\n",
"\n",
"with tiledb.SparseArray(tiledb_uri) as arr:\n",
" pts = arr.multi_index[x1:x2, y1:y2]\n",
" \n",
"print(f\"Retrieved {len(pts['longitude'])} vessels\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Turn the result into a Pandas dataframe"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df = pd.DataFrame(pts)\n",
"df.loc[:, 'x'], df.loc[:, 'y'] = lnglat_to_meters(df.longitude,df.latitude)\n",
"df"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"And visualize using datashader."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%time\n",
"\n",
"hv.output(backend=\"bokeh\")\n",
"bkgrd = tiles.EsriImagery().opts(xaxis=None, yaxis=None, width=700, height=500)\n",
"points = hv.Points(df, ['x', 'y'])\n",
"bkgrd * hd.datashade(points, cmap=cm.plasma)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.9"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
105 changes: 105 additions & 0 deletions geo/exactEarth/Ships_SQL.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Querying AIS data with serverless SQL"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import folium\n",
"from folium.plugins import MarkerCluster\n",
"import tiledb, tiledb.cloud\n",
"import time\n",
"\n",
"tiledb_uri = 'tiledb://exactEarth/ais_aug_2019'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"x1, x2, y1, y2 = [4.421052932739258, 4.437961578369141, 51.88973610006440, 51.89696629368307]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Query for vessels using SQL"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"q = f\"SELECT * from `{tiledb_uri}` WHERE `longitude` BETWEEN {x1} AND {x2} AND `latitude` BETWEEN {y1} and {y2} AND `vessel_name`='ADRI'\"\n",
"df = tiledb.cloud.sql.exec(query=q)\n",
"df"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"This is interesting but lets cluster the vessel location to identify hot spots and drill down."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"m = folium.Map(location=[51.899, 4.448], zoom_start=14)\n",
"\n",
"marker_cluster = MarkerCluster().add_to(m)\n",
"\n",
"# add a marker for every record in the filtered data, use a clustered view\n",
"for index, row in df.iterrows():\n",
" lat, lon = row['latitude'], row['longitude']\n",
" folium.Marker([lat, lon], name=row['vessel_name'], tooltip=row['vessel_name']).add_to(marker_cluster)\n",
"\n",
"m"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.8"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Loading

0 comments on commit 843eb65

Please sign in to comment.