Implement Pipeline script

leonelluiscorado · Feb 16, 2024 · 884386e · 884386e
1 parent 0acf4b5
commit 884386e
Show file tree

Hide file tree

Showing 8 changed files with 80 additions and 10 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,7 @@
 *.h5*
 *.json
 *GranuleList*
-*.Rhistory*
+*.Rhistory*
+*.gpkg*
+*.shp*
+*__pycache__*
diff --git a/gedi_pipeline.py b/gedi_pipeline.py
@@ -1,2 +1,59 @@
-import GEDIPipeline
+from pipeline.pipeline import GEDIPipeline
+
+import argparse
+
+# --------------------------COMMAND LINE ARGUMENTS AND ERROR HANDLING---------------------------- #
+# Set up argument and error handling
+parser = argparse.ArgumentParser(description='This pipeline performs all of the tasks (Finding, Downloading, Subsetting) to get GEDI Data.')
+
+parser.add_argument('--dir', required=True, help='Local directory to save GEDI files to be processed / Save the subsetted granules.')
+
+parser.add_argument('--product', required=True, help='GEDI Product to specify for the search query and subsetting module \
+                    Select from "GEDI01_B"; "GEDI02_A"; "GEDI02_B"; "GEDI04_A"')
+
+parser.add_argument('--version', required=True, help='GEDI Product version to specify for the search query and subsetting module \
+                    Select from "001"; "002"')
+
+parser.add_argument('--start', required=True, help='Start date for time period of interest: valid format is yyyy.mm.dd (e.g. 2020.11.12).')
+
+parser.add_argument('--end', required=True, help='Start date for time period of interest: valid format is yyyy.mm.dd (e.g. 2021.07.01).')
+
+parser.add_argument('--roi', required=True, help='Region of interest (ROI) to subset the GEDI orbit to in the output file. \
+                    Valid inputs are a geojson or .shp file or bounding box coordinates: ul_lat,ul_lon,lr_lat,lr_lon')
+
+parser.add_argument('--beams', required=False, help='Specific beams to be included in the output file (default is all beams) \
+                    BEAM0000,BEAM0001,BEAM0010,BEAM0011 are Coverage Beams. BEAM0101,BEAM0110,BEAM1000,BEAM1011 are Full Power Beams.', default=None)
+
+parser.add_argument('--sds', required=False, help='Specific science datasets (SDS) to include in the output subsetted file. \
+                    (see README for a list of available SDS and a list of default SDS returned for each product).', default=None)
+
+parser.add_argument('--login_keep', required=False, help='Include this option to keep EarthData login saved to this machine. It defaults saving to \
+                    the .netrc file', action='store_true')
+
+args = parser.parse_args()
+
+# ------------------------------------------------------------------------------------#
+
+pipeline = GEDIPipeline(
+    out_directory = args.dir,
+    product = args.product,
+    version = args.version,
+    date_start = args.start,
+    date_end = args.end,
+    roi = args.roi,
+    beams = args.beams,
+    sds = args.sds,
+    persist_login = args.login_keep
+)
+
+print("[Pipeline] Pipeline set, starting ...")
+
+try:
+    granules = pipeline.run_pipeline()
+except Exception as e:
+    print("[Pipeline] Failed to complete running the Pipeline. See the error below for more information.")
+    print(e)
+    exit(0)
+
+print(f"[Pipeline] Pipeline Run Complete! Subsetted {len(granules)} files saved to: {args.dir}")
 
diff --git a/pipeline/__init__.py b/pipeline/__init__.py
@@ -1 +0,0 @@
-from .gedi_pipeline import GEDIPipeline

diff --git a/pipeline/gedi_downloader.py → pipeline/downloader.py b/pipeline/gedi_downloader.py → pipeline/downloader.py
@@ -136,7 +136,7 @@ def download_granule(self, url, chunk_size=128):
 			self.__download(http_response.iter_content(chunk_size=chunk_size), file_path, response_length)
 
 		# Check file integrity / if it downloaded correctly
-		if not os.getsize(file_path) == int(response_length):
+		if not os.path.getsize(file_path) == int(response_length):
 			# If not downloaded correctly, send message for download retry
 			return False
 

diff --git a/pipeline/gedi_finder.py → pipeline/finder.py b/pipeline/gedi_finder.py → pipeline/finder.py
diff --git a/pipeline/gedi_pipeline.py → pipeline/pipeline.py b/pipeline/gedi_pipeline.py → pipeline/pipeline.py
@@ -1,22 +1,28 @@
-from pipeline import *
+from .finder import *
+from .subsetter import *
+from .downloader import *
 
 """
 Script that controls the entire GEDI Finder - Downloader - Subsetter pipeline.
 """
 
 class GEDIPipeline:
+    """
+    The GEDIPipeline :class: performs all operations in selecting, downloading and subsetting GEDI Data for a given region of interest
+    Args:
+        out
+    """
 
-    def __init__(self, username, password, out_directory, product, version, date_start, date_end, roi, sds, beams):
+    def __init__(self, out_directory, product, version, date_start, date_end, roi, sds, beams, persist_login=False):
 
         self.product = product
         self.version = version
         self.date_start, self.date_end = date_start, date_end
         self.roi = [float(c) for c in roi.split(",")]
-        self.username = username
-        self.password = password
         self.out_directory = out_directory
         self.sds = sds
         self.beams = beams
+        self.persist_login = persist_login
 
         self.finder = GEDIFinder(
             product=self.product,
@@ -27,8 +33,7 @@ def __init__(self, username, password, out_directory, product, version, date_sta
         )
 
         self.downloader = GEDIDownloader(
-            username=self.username,
-            password=self.password,
+            persist_login=self.persist_login,
             save_path=self.out_directory
         )
 
@@ -44,6 +49,7 @@ def __init__(self, username, password, out_directory, product, version, date_sta
         if not os.path.exists(out_directory):
             os.mkdir(out_directory)
 
+
     def run_pipeline(self):
 
         all_granules = self.finder.find(output_filepath=self.out_directory, save_file=True)

diff --git a/pipeline/gedi_subsetter.py → pipeline/subsetter.py b/pipeline/gedi_subsetter.py → pipeline/subsetter.py
diff --git a/pipeline/test.py b/pipeline/test.py
@@ -0,0 +1,5 @@
+from gedi_downloader import GEDIDownloader
+
+downloader = GEDIDownloader()
+
+downloader.download_granule(url="https://data.ornldaac.earthdata.nasa.gov/protected/gedi/GEDI_L4A_AGB_Density_V2_1/data/GEDI04_A_2019222172335_O03740_03_T02786_02_002_02_V002.h5")