Transition to pgstac backing (#1210)

* Begin transitioning to pgstac backing * Working collections endpoint * Working collections endpoint * Enable item read endpoints * Remove support for tiling; begin implementing /search * Use only pgstac * Reorganize commands around pgstac usage * Clean up dev environment for pgstac usage * Organize and clean up code * Search endpoint functional * Fix search links * Add collection type and serialization * Use model for items * Update item service to use search endpoint * Enable transactions * In situ * Fix json formatting, add transaction endpoints * Rename scriptrunner to pypgstac * Fix path * Lint * Lint * Fix pypgstac container paths * Add pre-commit and properly lint * Add sort functionality. * Fix temporal extent logic * Organize link updating * Fix feature links on search * Update links without code duplication * Organize link creation into queries to keep logic simple * Add rootlink in link updating functions * Add joplin data ingest * Added stac hierarchy model * Add hierarchy definition to service configuration * Format joplin data for ingest as example dataset * Working children and browseable structure * Remove references to NAIP outside of tests * Update documentation; add pgstac ADR * Fix validator errors * Revert timeinterval->temporal change * Enable more permissive cors headers * Lint * Appease scapegoat * Lint tests * Lint * Lint build.sbt * Remove unused dep check: it doesn't work * Fix docker build step * Thank you, github * Remove flyway references * Appease the fickle style gods Co-authored-by: Vijay Lulla <vijaylulla@gmail.com>
azavea · Sep 26, 2022 · 71dcdbd · 71dcdbd
1 parent e157a6a
commit 71dcdbd
Show file tree

Hide file tree

Showing 167 changed files with 4,752 additions and 7,575 deletions.
diff --git a/.gitignore b/.gitignore
@@ -142,4 +142,7 @@ main.bundle.js.map
 data/*
 
 # copilot related artifacts
-copilot/
+copilot/
+
+# metals
+**/metals.sbt
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,33 @@
+repos:
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+        language_version: python3
+        args: ["--safe"]
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        language_version: python3
+
+  - repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+        language_version: python3
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.960
+    hooks:
+      - id: mypy
+        exclude: ^tests/
+        args: [--strict]
+        additional_dependencies: ["pydantic", "types-jsonschema", "types-setuptools", "types-requests"]
+
+  - repo: https://github.com/koalaman/shellcheck-precommit
+    rev: v0.8.0
+    hooks:
+    - id: shellcheck
+      args: ["--severity=warning"]
diff --git a/application/Dockerfile b/application/Dockerfile
@@ -9,7 +9,7 @@ RUN \
       -u 1000 \
       franklin
 
-COPY ./target/scala-2.12/franklin-api-assembly.jar /var/lib/franklin/
+COPY ./application/target/scala-2.12/franklin-api-assembly.jar /var/lib/franklin/
 
 USER franklin
 WORKDIR /var/lib/franklin

diff --git a/application/Dockerfile.dev b/application/Dockerfile.dev
@@ -0,0 +1,16 @@
+FROM hseeberger/scala-sbt:8u222_1.3.5_2.13.1
+
+RUN \
+      addgroup --system franklin \
+      && adduser --system --disabled-login --disabled-password \
+      --home /var/lib/franklin \
+      --shell /sbin/nologin \
+      --ingroup franklin \
+      -u 1000 \
+      franklin
+
+USER franklin
+WORKDIR /opt/src
+
+
+CMD ["./sbt", '"project application"', '"run serve --db-host=pgstac"']
diff --git a/application/src/main/scala/com/azavea/franklin/api/FranklinJsonPrinter.scala b/application/src/main/scala/com/azavea/franklin/api/FranklinJsonPrinter.scala
@@ -0,0 +1,12 @@
+package com.azavea.franklin.api
+
+import io.circe.Printer
+import sttp.tapir.json.circe._
+
+// This custom json printing implementation for tapir is based
+//  on https://tapir.softwaremill.com/en/latest/endpoint/json.html#circe
+object FranklinJsonPrinter extends TapirJsonCirce {
+  // NOTE: the contents of this object should be imported for *tapir endpoints* rather than
+  //  http4s services. e.g. SearchEndpoints.scala but not SearchService.scala
+  override def jsonPrinter: Printer = Printer.spaces2.copy(dropNullValues = true)
+}
diff --git a/application/src/main/scala/com/azavea/franklin/api/Server.scala b/application/src/main/scala/com/azavea/franklin/api/Server.scala
@@ -2,18 +2,20 @@ package com.azavea.franklin.api
 
 import cats.effect._
 import cats.syntax.all._
-import com.azavea.franklin.api.commands.{ApiConfig, Commands, DatabaseConfig}
-import com.azavea.franklin.api.endpoints.LandingPageEndpoints
 import com.azavea.franklin.api.endpoints.{
+  CatalogEndpoints,
   CollectionEndpoints,
-  CollectionItemEndpoints,
-  SearchEndpoints,
-  TileEndpoints
+  ItemEndpoints,
+  LandingPageEndpoints,
+  SearchEndpoints
 }
 import com.azavea.franklin.api.middleware.AccessLoggingMiddleware
 import com.azavea.franklin.api.services._
-import com.azavea.franklin.extensions.validation.{collectionExtensionsRef, itemExtensionsRef}
-import com.azavea.stac4s.{`application/json`, StacLink, StacLinkType}
+import com.azavea.franklin.api.util.SwaggerHttp4s
+import com.azavea.franklin.commands._
+import com.azavea.franklin.datamodel._
+import com.azavea.franklin.datamodel.hierarchy._
+import com.azavea.stac4s.{`application/json`, StacLinkType}
 import com.google.common.util.concurrent.ThreadFactoryBuilder
 import doobie.hikari.HikariTransactor
 import doobie.util.ExecutionContexts
@@ -32,7 +34,6 @@ import sttp.client._
 import sttp.client.asynchttpclient.cats.AsyncHttpClientCatsBackend
 import sttp.tapir.docs.openapi.OpenAPIDocsInterpreter
 import sttp.tapir.openapi.circe.yaml._
-import sttp.tapir.swagger.http4s.SwaggerHttp4s
 
 import scala.concurrent.ExecutionContext
 
@@ -85,12 +86,6 @@ $$$$
       apiConfig: ApiConfig,
       dbConfig: DatabaseConfig
   ) = {
-    val rootLink = StacLink(
-      apiConfig.apiHost,
-      StacLinkType.StacRoot,
-      Some(`application/json`),
-      Some("Welcome to Franklin")
-    )
     implicit val logger = Slf4jLogger.getLogger[IO]
     AsyncHttpClientCatsBackend.resource[IO]() flatMap { implicit backend =>
       for {
@@ -101,58 +96,38 @@ $$$$
           connectionEc,
           Blocker.liftExecutionContext(transactionEc)
         )
-        collectionItemEndpoints = new CollectionItemEndpoints[IO](
-          apiConfig.defaultLimit,
+        collectionEndpoints = new CollectionEndpoints[IO](
           apiConfig.enableTransactions,
-          apiConfig.enableTiles,
           apiConfig.path
         )
-        collectionEndpoints = new CollectionEndpoints[IO](
+        itemEndpoints = new ItemEndpoints[IO](
+          apiConfig.defaultLimit,
           apiConfig.enableTransactions,
-          apiConfig.enableTiles,
           apiConfig.path
         )
-        landingPage = new LandingPageEndpoints[IO](apiConfig.path)
-        allEndpoints = collectionEndpoints.endpoints ++ collectionItemEndpoints.endpoints ++ new SearchEndpoints[
-          IO
-        ](apiConfig.path).endpoints ++ new TileEndpoints[
-          IO
-        ](
-          apiConfig.enableTiles,
-          apiConfig.path
-        ).endpoints ++ landingPage.endpoints
-        docs      = OpenAPIDocsInterpreter.toOpenAPI(allEndpoints, "Franklin", "0.0.1")
-        docRoutes = new SwaggerHttp4s(docs.toYaml, "open-api", "spec.yaml").routes[IO]
-        searchRoutes = new SearchService[IO](
-          apiConfig,
-          apiConfig.defaultLimit,
-          apiConfig.enableTiles,
-          xa,
-          rootLink
-        ).routes
-        tileRoutes = new TileService[IO](
-          apiConfig.apiHost,
-          apiConfig.enableTiles,
-          apiConfig.path,
-          xa
-        ).routes
-        itemExtensions       <- Resource.eval { itemExtensionsRef[IO] }
-        collectionExtensions <- Resource.eval { collectionExtensionsRef[IO] }
-        collectionRoutes = new CollectionsService[IO](xa, apiConfig, collectionExtensions).routes <+> new CollectionItemsService[
-          IO
-        ](
-          xa,
-          apiConfig,
-          itemExtensions,
-          rootLink
-        ).routes
+        searchEndpoints  = new SearchEndpoints[IO](apiConfig)
+        catalogEndpoints = new CatalogEndpoints[IO](apiConfig.path)
+        landingPage      = new LandingPageEndpoints[IO](apiConfig.path)
+        allEndpoints = collectionEndpoints.endpoints ++
+          catalogEndpoints.endpoints ++
+          itemEndpoints.endpoints ++
+          searchEndpoints.endpoints ++
+          landingPage.endpoints
+        docs              = OpenAPIDocsInterpreter.toOpenAPI(allEndpoints, "Franklin", "0.0.1")
+        docRoutes         = new SwaggerHttp4s(docs.toYaml, "open-api", "spec.yaml").routes[IO]
+        collectionRoutes  = new CollectionService[IO](apiConfig, xa).routes
+        catalogRoutes     = new CatalogService[IO](apiConfig).routes
+        itemRoutes        = new ItemService[IO](apiConfig, xa).routes
+        searchRoutes      = new SearchService[IO](apiConfig, xa).routes
         landingPageRoutes = new LandingPageService[IO](apiConfig).routes
-        router = CORS(
-          new AccessLoggingMiddleware(
-            collectionRoutes <+> searchRoutes <+> tileRoutes <+> landingPageRoutes <+> docRoutes,
-            logger
-          ).withLogging(true)
-        ).orNotFound
+        router = CORS.policy
+          .withAllowOriginAll(
+            new AccessLoggingMiddleware(
+              collectionRoutes <+> catalogRoutes <+> itemRoutes <+> searchRoutes <+> landingPageRoutes <+> docRoutes,
+              logger
+            ).withLogging(true)
+          )
+          .orNotFound
         serverBuilderBlocker <- Blocker[IO]
         server <- {
           BlazeServerBuilder[IO](serverBuilderBlocker.blockingContext)
@@ -172,32 +147,10 @@ $$$$
     import Commands._
 
     applicationCommand.parse(args, env = sys.env) map {
-      case RunServer(apiConfig, dbConfig) if !apiConfig.runMigrations =>
+      case RunServer(apiConfig, dbConfig) =>
         createServer(apiConfig, dbConfig)
           .use(_ => IO.never)
           .as(ExitCode.Success)
-      case RunServer(apiConfig, dbConfig) =>
-        runMigrations(dbConfig) *>
-          createServer(apiConfig, dbConfig).use(_ => IO.never).as(ExitCode.Success)
-      case RunMigrations(config) => runMigrations(config)
-      case RunCatalogImport(catalogRoot, dbConfig, dryRun) =>
-        AsyncHttpClientCatsBackend.resource[IO]() use { implicit backend =>
-          runCatalogImport(catalogRoot, dbConfig, dryRun) map { _ => ExitCode.Success }
-        }
-      case RunItemsImport(collectionId, itemUris, dbConfig, dryRun) => {
-        AsyncHttpClientCatsBackend.resource[IO]() use { implicit backend =>
-          runStacItemImport(collectionId, itemUris, dbConfig, dryRun) map {
-            case Left(error) => {
-              println(s"Import failed: $error")
-              ExitCode.Error
-            }
-            case Right(numItemsImported) => {
-              println(s"Import succesful: ${numItemsImported} items imported")
-              ExitCode.Success
-            }
-          }
-        }
-      }
     } match {
       case Left(e) =>
         IO {

diff --git a/application/src/main/scala/com/azavea/franklin/api/endpoints/CatalogEndpoints.scala b/application/src/main/scala/com/azavea/franklin/api/endpoints/CatalogEndpoints.scala
@@ -0,0 +1,34 @@
+package com.azavea.franklin.api.endpoints
+
+import cats.effect.Concurrent
+import com.azavea.franklin.api.FranklinJsonPrinter._
+import com.azavea.franklin.api.schemas._
+import com.azavea.franklin.datamodel.{Catalog, CollectionsResponse}
+import com.azavea.franklin.error.NotFound
+import io.circe._
+import sttp.capabilities.fs2.Fs2Streams
+import sttp.model.StatusCode
+import sttp.model.StatusCode.{NotFound => NF}
+import sttp.tapir._
+import sttp.tapir.generic.auto._
+
+import java.util.UUID
+
+class CatalogEndpoints[F[_]: Concurrent](
+    pathPrefix: Option[String]
+) {
+
+  val basePath = baseFor(pathPrefix, "catalogs")
+
+  val base = endpoint.in(basePath)
+
+  val catalogUnique: Endpoint[List[String], NotFound, Catalog, Fs2Streams[F]] =
+    base.get
+      .in(paths)
+      .out(jsonBody[Catalog])
+      .errorOut(oneOf(statusMapping(NF, jsonBody[NotFound].description("not found"))))
+      .description("A single catalog")
+      .name("catalogUnique")
+
+  val endpoints = List(catalogUnique)
+}