Test_GeoMAD_Landsat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datacube.utils.dask import start_local_dask\n",
    "from dep_tools.loaders import LandsatOdcLoader\n",
    "from dep_tools.namers import LocalPath\n",
    "from dep_tools.writers import LocalDsWriter\n",
    "\n",
    "from src.run_task import GeoMADLandsatProcessor, get_grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optionally set up a local dask cluster\n",
    "client = start_local_dask()\n",
    "client.dashboard_link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Study site configuration\n",
    "region_code = \"66,19\"\n",
    "\n",
    "# Issues\n",
    "# Cloudy island\n",
    "# https://deppcpublicstorage.blob.core.windows.net/output/dep_ls_geomad/0-1-0/099/022/2023/dep_ls_geomad_099_022_2023.stac-item.json\n",
    "region_code = \"99,22\"\n",
    "# Noisy tile\n",
    "# https://deppcpublicstorage.blob.core.windows.net/output/dep_ls_geomad/0-1-0/063/020/2023/dep_ls_geomad_063_020_2023.stac-item.json\n",
    "region_code=\"63,20\"\n",
    "\n",
    "\n",
    "datetime = \"2023\"\n",
    "item_id = region_code\n",
    "\n",
    "# And get the study site\n",
    "grid = get_grid()\n",
    "cell = grid.loc[[(region_code)]]\n",
    "cell.explore()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up a data loader\n",
    "loader = LandsatOdcLoader(\n",
    "    epsg=3832,\n",
    "    datetime=datetime,\n",
    "    dask_chunksize=dict(time=1, x=4096, y=4096),\n",
    "    odc_load_kwargs=dict(\n",
    "        fail_on_error=False,\n",
    "        resolution=90,\n",
    "        # bands=[\"qa_pixel\", \"red\", \"green\", \"blue\", \"nir08\", \"swir16\", \"swir22\"],\n",
    "        groupby=\"solar_day\",\n",
    "        bands=[\"qa_pixel\", \"red\", \"green\", \"blue\"]\n",
    "    ),\n",
    "    exclude_platforms=[\"landsat-7\"],\n",
    "    nodata_value=0,\n",
    "    keep_ints=True,\n",
    "    load_as_dataset=True,\n",
    "    only_tier_one=True,\n",
    "    fall_back_to_tier_two=True\n",
    ")\n",
    "\n",
    "# Run the load process, which is lazy-loaded\n",
    "input_data = loader.load(cell)\n",
    "input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set up a data processor\n",
    "processor = GeoMADLandsatProcessor(\n",
    "    scale_and_offset=False,\n",
    "    harmonize_to_old=True,\n",
    "    geomad_options=dict(\n",
    "        work_chunks=(601, 601),\n",
    "        num_threads=10,\n",
    "    ),\n",
    "    filters=[(\"closing\", 5), (\"opening\", 5)],\n",
    "    keep_ints=True\n",
    ")\n",
    "\n",
    "# Plan the processing. Actually loads the data.\n",
    "output_data = processor.process(input_data)\n",
    "output_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_data[[\"red\", \"green\", \"blue\"]].squeeze().to_array().plot.imshow(size=10, vmin=7000, vmax=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_data[\"count\"].plot.imshow(size=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Testing the Azure writer\n",
    "\n",
    "# from dep_tools.writers import AzureDsWriter\n",
    "# from dep_tools.namers import DepItemPath\n",
    "\n",
    "# itempath = DepItemPath(\"geomad\", \"test\", \"0.0\", datetime)\n",
    "\n",
    "# writer = AzureDsWriter(\n",
    "#     itempath=itempath,\n",
    "#     overwrite=True,\n",
    "#     convert_to_int16=False,\n",
    "#     extra_attrs=dict(dep_version=\"0.0\"),\n",
    "# )\n",
    "\n",
    "# writer.write(output_data, \"test\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from odc.stac import load\n",
    "from pystac import Item\n",
    "\n",
    "item = Item.from_file(\"https://deppcpublicstorage.blob.core.windows.net/output/dep_geomad_test/0-0/test/2023-01/dep_geomad_test_test_2023-01.stac-item.json\")\n",
    "\n",
    "data = load([item], chunks={})\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.red.isel(time=0).plot.imshow(size=8, robust=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is the target path\n",
    "dep_path = LocalPath(\n",
    "    local_folder=\"data\",\n",
    "    sensor=\"ls\",\n",
    "    dataset_id=\"geomad\",\n",
    "    version=\"0.0.0\",\n",
    "    time=datetime,\n",
    "    zero_pad_numbers=True\n",
    ")\n",
    "\n",
    "# Set up a writer and write out the files\n",
    "writer = LocalDsWriter(\n",
    "    itempath=dep_path,\n",
    "    output_nodata=0,\n",
    "    use_odc_writer=True,\n",
    "    overwrite=True,\n",
    "    convert_to_int16=False\n",
    ")\n",
    "print(f\"Writing to: {dep_path._folder(item_id)}\")\n",
    "out_files = writer.write(loaded, item_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make sure written files are readable\n",
    "stac_path = writer.itempath.path(item_id, ext=\".stac-item.json\")\n",
    "\n",
    "item = Item.from_file(stac_path)\n",
    "item.validate()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}