Ensure correct footprints and tasks for pre-COGs (#5627)

raster-foundry · Sep 16, 2021 · 2497581 · 2497581
1 parent ce39ffa
commit 2497581
Show file tree

Hide file tree

Showing 7 changed files with 63 additions and 122 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,8 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- Corrected task side length calculation for non-Web Mercator COGs [#5627](https://github.com/raster-foundry/raster-foundry/pull/5627)
 
 ## [1.68.0] - 2021-09-13
 ### Changed

diff --git a/app-tasks/rf/src/rf/commands/process_upload.py b/app-tasks/rf/src/rf/commands/process_upload.py
@@ -1,14 +1,15 @@
 import logging
 import os
+import time
 
 import click
 from planet import api
 
-from ..models import Upload, AnnotationProject
+from ..models import Upload, Scene, AnnotationProject
 from ..uploads.geotiff import GeoTiffS3SceneFactory
 from ..uploads.geotiff.io import update_annotation_project
 from ..utils.exception_reporting import wrap_rollbar
-from ..utils.io import get_session, notify_intercom, copy_to_debug
+from ..utils.io import get_session, notify_intercom, copy_to_debug, IngestStatus
 
 logger = logging.getLogger(__name__)
 HOST = os.getenv("RF_HOST")
@@ -19,6 +20,19 @@ class TaskGridError(Exception):
     pass
 
 
+def wait_for_scene(scene_id, attempt, max_attempts):
+    scene = Scene.from_id(scene_id)
+    if scene.ingestStatus != IngestStatus.INGESTED and attempt < max_attempts - 1:
+        time.sleep(3)
+        return wait_for_scene(scene_id, attempt + 1, max_attempts)
+    elif scene.ingestStatus == IngestStatus.INGESTED:
+        return
+    else:
+        raise Exception(
+            "Scene did not become ingested in a reasonable amount of time. If this completes eventually, you can resume upload processing later at the next step"
+        )
+
+
 @click.command(name="process-upload")
 @click.argument("upload_id")
 @wrap_rollbar
@@ -106,6 +120,9 @@ def process_upload(upload_id):
         )
 
         generate_tasks = upload.annotationProjectId is not None and upload.generateTasks
+        for new_scene_id in scene_ids:
+            wait_for_scene(new_scene_id, 0, 10)
+
         if generate_tasks:
             try:
                 [

diff --git a/app-tasks/rf/src/rf/models/scene.py b/app-tasks/rf/src/rf/models/scene.py
@@ -4,10 +4,10 @@
 from requests.exceptions import HTTPError
 import logging
 
-from .base import BaseModel
-from .thumbnail import Thumbnail
-from .image import Image
-from .footprint import Footprint
+from rf.models.base import BaseModel
+from rf.models.thumbnail import Thumbnail
+from rf.models.image import Image
+from rf.models.footprint import Footprint
 
 logger = logging.getLogger(__name__)
 

diff --git a/app-tasks/rf/src/rf/uploads/geotiff/create_scenes.py b/app-tasks/rf/src/rf/uploads/geotiff/create_scenes.py
@@ -1,9 +1,8 @@
 import logging
 import uuid
 
-from rf.models import Footprint, Scene
+from rf.models import Scene
 from rf.utils.io import IngestStatus, JobStatus, Visibility
-from rf.utils.footprint import complex_footprint
 from shapely.geometry import mapping, MultiPolygon, Polygon
 
 from .io import get_geotiff_metadata, get_geotiff_name
@@ -65,8 +64,6 @@ def create_geotiff_scene(
     }
     # Override defaults with kwargs
     sceneKwargs.update(kwargs)
-    data_footprint = complex_footprint(tif_path)
-    tile_footprint = MultiPolygon([Polygon.from_bounds(*data_footprint.bounds)])
 
     # Construct Scene
     scene = Scene(
@@ -81,8 +78,8 @@ def create_geotiff_scene(
         metadataFiles,
         owner=owner,
         sceneType=sceneType,
-        dataFootprint=Footprint(mapping(data_footprint)["coordinates"]),
-        tileFootprint=Footprint(mapping(tile_footprint)["coordinates"]),
+        dataFootprint=None,
+        tileFootprint=None,
         **sceneKwargs
     )
 

diff --git a/app-tasks/rf/src/rf/uploads/geotiff/io.py b/app-tasks/rf/src/rf/uploads/geotiff/io.py
@@ -6,6 +6,7 @@
 
 
 from rf.models import AnnotationProject
+from rf.utils.io import get_tempdir
 
 
 def get_geotiff_metadata(tif_path):
@@ -43,8 +44,17 @@ def get_geotiff_resolution(tif_path):
     Returns:
         Resolution of the GeoTIFF as an (x, y) tuple.
     """
-    with rasterio.open(tif_path) as src:
-        return src.res
+    with get_tempdir() as tempdir:
+        warped_path = os.path.join(tempdir, "warped-web-mercator.vrt")
+        subprocess.check_call([
+            "gdalwarp",
+            "-t_srs",
+            "epsg:3857",
+            tif_path.replace("s3://", "/vsis3/"),
+            warped_path
+        ])
+        with rasterio.open(warped_path) as src:
+            return src.res
 
 
 def get_geotiff_size_bytes(tif_path):

diff --git a/app-tasks/rf/src/rf/utils/footprint.py b/app-tasks/rf/src/rf/utils/footprint.py
diff --git a/app-tasks/rf/src/rf/utils/io.py b/app-tasks/rf/src/rf/utils/io.py
@@ -22,6 +22,29 @@
 data_bucket = os.getenv("DATA_BUCKET")
 
 
+class JobStatus(object):
+    QUEUED = "QUEUED"
+    PROCESSING = "PROCESSING"
+    FAILURE = "FAILURE"
+    SUCCESS = "SUCCESS"
+    UPLOADING = "UPLOADING"
+    PARTIALFAILURE = "PARTIALFAILURE"
+
+
+class IngestStatus(object):
+    NOTINGESTED = "NOTINGESTED"
+    TOBEINGESTED = "TOBEINGESTED"
+    INGESTING = "INGESTING"
+    INGESTED = "INGESTED"
+    FAILED = "FAILED"
+
+
+class Visibility(object):
+    PUBLIC = "PUBLIC"
+    ORGANIZATION = "ORGANIZATION"
+    PRIVATE = "PRIVATE"
+
+
 @contextmanager
 def get_tempdir(debug=False):
     """Returns a temporary directory that is cleaned up after usage
@@ -271,26 +294,3 @@ def copy_to_debug(upload):
             Key=source_key,
             CopySource={"Bucket": source_bucket, "Key": source_key},
         )
-
-
-class JobStatus(object):
-    QUEUED = "QUEUED"
-    PROCESSING = "PROCESSING"
-    FAILURE = "FAILURE"
-    SUCCESS = "SUCCESS"
-    UPLOADING = "UPLOADING"
-    PARTIALFAILURE = "PARTIALFAILURE"
-
-
-class IngestStatus(object):
-    NOTINGESTED = "NOTINGESTED"
-    TOBEINGESTED = "TOBEINGESTED"
-    INGESTING = "INGESTING"
-    INGESTED = "INGESTED"
-    FAILED = "FAILED"
-
-
-class Visibility(object):
-    PUBLIC = "PUBLIC"
-    ORGANIZATION = "ORGANIZATION"
-    PRIVATE = "PRIVATE"