Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Transition to pgstac backing #1210

Merged
merged 53 commits into from
Sep 26, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
53 commits
Select commit Hold shift + click to select a range
fe6fb05
Begin transitioning to pgstac backing
moradology May 11, 2022
dfd654d
Working collections endpoint
moradology May 12, 2022
0115155
Working collections endpoint
moradology May 11, 2022
48e9a1d
Enable item read endpoints
moradology May 16, 2022
c39eeae
Merge branch 'feature/pgstac-backing' of github.com:azavea/franklin i…
moradology May 16, 2022
b85d609
Remove support for tiling; begin implementing /search
moradology May 17, 2022
0f64f86
Use only pgstac
moradology May 17, 2022
f01e85f
Reorganize commands around pgstac usage
moradology May 18, 2022
9234a14
Clean up dev environment for pgstac usage
moradology May 18, 2022
f7d2c8d
Organize and clean up code
moradology May 19, 2022
84199c5
Search endpoint functional
moradology May 24, 2022
e2b502c
Fix search links
moradology May 24, 2022
bcfbc8e
Add collection type and serialization
moradology May 31, 2022
8aac372
Use model for items
moradology May 31, 2022
f8cdce3
Update item service to use search endpoint
moradology May 31, 2022
e7bd9d4
Enable transactions
moradology Jun 1, 2022
b393219
In situ
moradology Jun 1, 2022
6d18c1a
Fix json formatting, add transaction endpoints
moradology Jun 2, 2022
94542df
Rename scriptrunner to pypgstac
moradology Jun 2, 2022
aab666b
Fix path
moradology Jun 2, 2022
ef443bc
Lint
moradology Jun 2, 2022
0bb0788
Lint
moradology Jun 2, 2022
2839c54
Fix pypgstac container paths
moradology Jun 2, 2022
9170127
Add pre-commit and properly lint
moradology Jun 2, 2022
177914f
Add sort functionality.
vlulla Jun 2, 2022
bcdac26
Fix temporal extent logic
moradology Jun 7, 2022
476cfe6
Organize link updating
moradology Jun 7, 2022
767b57f
Fix feature links on search
moradology Jun 7, 2022
03679c0
Update links without code duplication
moradology Jun 7, 2022
c841090
Organize link creation into queries to keep logic simple
moradology Jun 14, 2022
da40986
Add rootlink in link updating functions
moradology Jun 15, 2022
9d6ebc0
Add joplin data ingest
moradology Jun 16, 2022
296611e
Added stac hierarchy model
moradology Jun 16, 2022
7d41f74
Add hierarchy definition to service configuration
moradology Jun 21, 2022
2311224
Format joplin data for ingest as example dataset
moradology Jun 21, 2022
4a0f7f7
Working children and browseable structure
moradology Jun 21, 2022
8bf46a9
Remove references to NAIP outside of tests
moradology Jun 21, 2022
566c39c
Update documentation; add pgstac ADR
moradology Jul 20, 2022
c837f40
Fix validator errors
moradology Aug 24, 2022
3fca246
Revert timeinterval->temporal change
moradology Aug 24, 2022
a22934d
Enable more permissive cors headers
moradology Aug 25, 2022
7184919
Lint
moradology Sep 15, 2022
d96b800
Appease scapegoat
moradology Sep 15, 2022
812aec5
Lint tests
moradology Sep 15, 2022
98d8dd1
Lint
moradology Sep 15, 2022
d74eb02
Lint build.sbt
moradology Sep 15, 2022
0638d15
Remove unused dep check: it doesn't work
moradology Sep 15, 2022
8c09a6f
Fix docker build step
moradology Sep 15, 2022
1773f96
Merge branch 'master' into feature/pgstac-backing
moradology Sep 15, 2022
6c8524b
Thank you, github
moradology Sep 15, 2022
33467a1
Remove flyway references
moradology Sep 15, 2022
ab7eb84
Appease the fickle style gods
moradology Sep 15, 2022
b1b0fdd
Merge branch 'master' into feature/pgstac-backing
moradology Sep 26, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Fix json formatting, add transaction endpoints
  • Loading branch information
moradology committed Jun 2, 2022
commit 6d18c1a78ce7d7fd96d9b350842c119e209be32f
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package com.azavea.franklin.api

import sttp.tapir.json.circe._
import io.circe.Printer


// This custom json printing implementation for tapir is based
// on https://tapir.softwaremill.com/en/latest/endpoint/json.html#circe
object FranklinJsonPrinter extends TapirJsonCirce {
// NOTE: the contents of this object should be imported for *tapir endpoints* rather than
// http4s services e.g. SearchEndpoints.scala but not SearchService.scala
override def jsonPrinter: Printer = Printer.spaces2.copy(dropNullValues = true)
}
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
package com.azavea.franklin.api.endpoints

import cats.effect.Concurrent
import com.azavea.franklin.api.FranklinJsonPrinter._
import com.azavea.franklin.api.schemas._
import com.azavea.franklin.datamodel.{CollectionsResponse, MosaicDefinition}
import com.azavea.franklin.datamodel.stactypes.{Collection}
import com.azavea.franklin.error.NotFound
import com.azavea.stac4s.StacCollection

import cats.effect.Concurrent
import io.circe._
import sttp.capabilities.fs2.Fs2Streams
import sttp.model.StatusCode
import sttp.model.StatusCode.{NotFound => NF}
import sttp.tapir._
import sttp.tapir.generic.auto._
import sttp.tapir.json.circe._

import java.util.UUID

Expand All @@ -25,6 +25,14 @@ class CollectionEndpoints[F[_]: Concurrent](

val base = endpoint.in(basePath)

val collectionUnique: Endpoint[String, NotFound, Collection, Fs2Streams[F]] =
base.get
.in(path[String])
.out(jsonBody[Collection])
.errorOut(oneOf(statusMapping(NF, jsonBody[NotFound].description("not found"))))
.description("A single collection")
.name("collectionUnique")

val collectionsList: Endpoint[Unit, Unit, CollectionsResponse, Fs2Streams[F]] =
base.get
.out(jsonBody[CollectionsResponse])
Expand Down Expand Up @@ -54,19 +62,11 @@ class CollectionEndpoints[F[_]: Concurrent](
.name("putCollection")

val deleteCollection: Endpoint[String, NotFound, Unit, Fs2Streams[F]] =
base.get
base.delete
.in(path[String])
.errorOut(oneOf(statusMapping(NF, jsonBody[NotFound].description("not found"))))
.name("collectionDelete")

val collectionUnique: Endpoint[String, NotFound, Collection, Fs2Streams[F]] =
base.get
.in(path[String])
.out(jsonBody[Collection])
.errorOut(oneOf(statusMapping(NF, jsonBody[NotFound].description("not found"))))
.description("A single collection")
.name("collectionUnique")

val endpoints = List(collectionsList, collectionUnique) ++ {
if (enableTransactions) List(postCollection, putCollection, deleteCollection) else Nil
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.azavea.franklin.api.endpoints

import cats.effect.Concurrent
import com.azavea.franklin.api.FranklinJsonPrinter._
import com.azavea.franklin.api.schemas._
import com.azavea.franklin.datamodel.IfMatchMode
import com.azavea.franklin.datamodel.PaginationToken
Expand All @@ -11,7 +11,9 @@ import com.azavea.franklin.error.{
NotFound,
ValidationError
}

import com.azavea.stac4s.StacItem
import cats.effect.Concurrent
import eu.timepit.refined.types.numeric.NonNegInt
import io.circe.{Codec => _, _}
import sttp.capabilities.fs2.Fs2Streams
Expand All @@ -20,7 +22,7 @@ import sttp.model.StatusCode.{NotFound => NF, BadRequest, PreconditionFailed}
import sttp.tapir._
import sttp.tapir.codec.refined._
import sttp.tapir.generic.auto._
import sttp.tapir.json.circe._


class ItemEndpoints[F[_]: Concurrent](
defaultLimit: NonNegInt,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
package com.azavea.franklin.api.endpoints

import com.azavea.franklin.api.FranklinJsonPrinter._
import com.azavea.franklin.api.schemas._
import com.azavea.franklin.database._
import com.azavea.franklin.datamodel.{PaginationToken, SearchParameters}
import com.azavea.franklin.datamodel.{PaginationToken, SearchParameters, StacSearchCollection}
import com.azavea.franklin.commands.ApiConfig
import com.azavea.stac4s.{Bbox, TemporalExtent}

import com.azavea.stac4s.{Bbox, TemporalExtent}
import cats.effect._
import eu.timepit.refined.types.numeric.NonNegInt
import geotrellis.vector.Geometry
Expand All @@ -16,7 +17,6 @@ import sttp.capabilities.fs2.Fs2Streams
import sttp.tapir._
import sttp.tapir.codec.refined._
import sttp.tapir.generic.auto._
import sttp.tapir.json.circe._

class SearchEndpoints[F[_]: Concurrent](apiConfig: ApiConfig) {

Expand Down Expand Up @@ -97,19 +97,19 @@ class SearchEndpoints[F[_]: Concurrent](apiConfig: ApiConfig) {
val searchPostInput: EndpointInput[SearchParameters] =
jsonBody[SearchParameters]

val searchGet: Endpoint[SearchParameters, Unit, Json, Fs2Streams[F]] =
val searchGet: Endpoint[SearchParameters, Unit, StacSearchCollection, Fs2Streams[F]] =
base
.get
.in(searchParameters)
.out(jsonBody[Json])
.out(jsonBody[StacSearchCollection])
.description("Search endpoint for all collections")
.name("search-get")

val searchPost: Endpoint[SearchParameters, Unit, Json, Fs2Streams[F]] =
val searchPost: Endpoint[SearchParameters, Unit, StacSearchCollection, Fs2Streams[F]] =
base
.post
.in(searchPostInput)
.out(jsonBody[Json])
.out(jsonBody[StacSearchCollection])
.description("Search endpoint using POST for all collections")
.name("search-post")

Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
package com.azavea.franklin.api.services

import cats.data.EitherT
import cats.effect._
import cats.effect.concurrent.Ref
import cats.syntax.all._
import com.azavea.franklin.commands.ApiConfig
import com.azavea.franklin.api.endpoints.CollectionEndpoints
import com.azavea.franklin.api.implicits._
import com.azavea.franklin.database.PGStacQueries
import com.azavea.franklin.datamodel.{CollectionsResponse, MosaicDefinition, TileInfo}
import com.azavea.franklin.datamodel.{CollectionsResponse, MosaicDefinition, TileInfo, Link}
import com.azavea.franklin.datamodel.stactypes.Collection
import com.azavea.franklin.error.{NotFound => NF}
import com.azavea.franklin.extensions.validation._
import com.azavea.stac4s._

import cats.data.EitherT
import cats.effect._
import cats.effect.concurrent.Ref
import cats.syntax.all._
import doobie._
import doobie.implicits._
import doobie.util.transactor.Transactor
Expand All @@ -31,12 +31,13 @@ import java.net.{URLDecoder, URLEncoder}
import java.nio.charset.StandardCharsets
import java.util.UUID


case class AddCollectionLinks(apiConfig: ApiConfig) {

val _collectionId = root.id.string

def createSelfLink(collection: Collection): StacLink = {
StacLink(
def createSelfLink(collection: Collection): Link = {
Link(
s"${apiConfig.apiHost}/collections/${collection.id}",
StacLinkType.Self,
Some(`application/json`),
Expand Down Expand Up @@ -70,8 +71,8 @@ class CollectionsService[F[_]: Concurrent](
collections <- PGStacQueries.listCollections().transact(xa)
collectionsWithLinks = collections.map(addCollectionLinks(_))
} yield {
val childLinks: List[StacLink] = collectionsWithLinks.map({ coll: Collection =>
StacLink(
val childLinks: List[Link] = collectionsWithLinks.map({ coll: Collection =>
Link(
s"${apiConfig.apiHost}/collections/${coll.id}",
StacLinkType.Child,
Some(`application/json`),
Expand Down Expand Up @@ -114,50 +115,17 @@ class CollectionsService[F[_]: Concurrent](
Right(collection)
}
}
// {
// val newCollection = collection.copy(links =
// collection.links.filter({ link =>
// !Set[StacLinkType](StacLinkType.Item, StacLinkType.StacRoot, StacLinkType.Self)
// .contains(link.rel)
// }) ++
// List(
// StacLink(
// s"$apiHost/collections/${collection.id}",
// StacLinkType.Self,
// Some(`application/json`),
// collection.title
// ),
// StacLink(
// s"$apiHost",
// StacLinkType.StacRoot,
// Some(`application/json`),
// None
// )
// )
// )
// for {
// inserted <- StacCollectionDao.insertStacCollection(newCollection, None).transact(xa)
// validator <- makeCollectionValidator(inserted.stacExtensions, collectionExtensionsRef)
// } yield Right(validator(inserted).asJson.dropNullValues)
// }

def deleteCollection(rawCollectionId: String): F[Either[NF, Unit]] = ???
// {
// val collectionId = URLDecoder.decode(rawCollectionId, StandardCharsets.UTF_8.toString)
// for {
// collectionOption <- StacCollectionDao
// .getCollection(collectionId)
// .transact(xa)
// deleted <- collectionOption traverse { _ =>
// StacCollectionDao.query.filter(fr"id = $collectionId").delete.transact(xa).void
// }
// } yield {
// Either.fromOption(
// deleted,
// NF(s"Collection $collectionId not found")
// )
// }
// }

def deleteCollection(rawCollectionId: String): F[Either[NF, Unit]] = {
val collectionId = URLDecoder.decode(rawCollectionId, StandardCharsets.UTF_8.toString)
try {
for {
queryResult <- PGStacQueries.deleteCollection(collectionId).attempt.transact(xa)
} yield {
queryResult.leftMap { _ => NF(s"Collection ID $collectionId not found") }
}
}
}

val collectionEndpoints =
new CollectionEndpoints[F](enableTransactions, apiConfig.path)
Expand Down
Loading