centralized rabbitmq, ocpdb update (#291)

* centralized rabbitmq, ocpdb update * prepare changelog for OCPDB and rabbitmq * add ocpdb config * new init mechanism * Update to 1.2.1 Co-authored-by: Jannis R #38c3 <mail@jannisr.de> --------- Co-authored-by: Jannis R #38c3 <mail@jannisr.de>
mobidata-bw · Jan 3, 2025 · f6e73a8 · f6e73a8
1 parent 6206bed
commit f6e73a8
Show file tree

Hide file tree

Showing 6 changed files with 116 additions and 31 deletions.
diff --git a/.env b/.env
@@ -99,23 +99,26 @@ GEOSERVER_MAXIMUM_MEMORY=4G
 # In prod, should switch to PRODUCTION_LOGGING. For other logging profiles see: https://docs.geoserver.org/stable/en/user/configuration/globalsettings.html#config-globalsettings-log-profile
 GEOSERVER_LOGGING_PROFILE=DEFAULT_LOGGING
 
+# RabbitMQ for OCPDB and ParkAPI
+RABBITMQ_VERSION=rabbitmq:3.13
+
 # ocpdb
 OCPDB_API_PORT=7000
 OCPDB_POSTGRES_USER=ocpdb
 OCPDB_POSTGRES_DB=ocpdb
 OCPDB_POSTGRES_HOST=ocpdb-db
-OCPDB_CELERY_BROKER_URL=amqp://ocpdb-rabbitmq
+OCPDB_CELERY_BROKER_URL=amqp://rabbitmq/ocpdb
 OCPDB_PRIVACY_URL=https://www.nvbw.de/datenschutz
 OCPDB_LEGAL_NOTICE_URL=https://www.nvbw.de/impressum
-OCPDB_IMAGE=ghcr.io/binary-butterfly/ocpdb:1.0.0
+OCPDB_IMAGE=ghcr.io/binary-butterfly/ocpdb:1.2.1
 OCPDB_DB_IMAGE=postgis/postgis:15-3.4-alpine
 
 # ParkAPI
 PARK_API_API_PORT=7500
 PARK_API_POSTGRES_USER=park-api
 PARK_API_POSTGRES_DB=park-api
 PARK_API_POSTGRES_HOST=park-api-db
-PARK_API_CELERY_BROKER_URL=amqp://park-api-rabbitmq
+PARK_API_CELERY_BROKER_URL=amqp://rabbitmq/park-api
 PARK_API_IMAGE=ghcr.io/parkendd/park-api-v3:0.16.2
 PARK_API_DB_IMAGE=postgis/postgis:15-3.4-alpine
 

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,7 +7,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 ## Unreleased
 
 ### Added
-- GeoServer: added ferry and furnicular to `mdbw_transit_stations_default` and `mdbw_transit_routes_default` gtfs styles 
+- GeoServer: added ferry and funicular to `mdbw_transit_stations_default` and `mdbw_transit_routes_default` gtfs styles
+- OCPDB 1.2 with Stadtwerke Stuttgart
+- OCPDB 1.2 with cron mechanisms for permanent data pulling
+
+### Changed
+
+- Centralize rabbitmq for better performance
+- Improve rabbitmq healthcheck for better performance
+- upgrade OCPDB to 1.2.1 with BNetzA Import Fix
+
 
 ## 2024-12-20
 
@@ -18,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 ### Changed
 - remove obsolete GBFS providers tier_basel, tier_bern, tier_stgallen, tier_winterthur, tier_zurich
 
+
 ## 2024-12-11
 
 ### Added
@@ -26,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 ### Changed
 - VRS parking providers obtain new Mobilithek subscription ids
 
+
 ## 2024-12-10
 
 ### Added
@@ -36,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 
 - new namespace for GBFS feed voi_de to avoid conflict with voi_ch
 
+
 ## 2024-12-04
 
 ### Changed

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -18,7 +18,7 @@ x-ocpdb-defaults: &ocpdb-defaults
   depends_on:
     ocpdb-db:
       condition: service_healthy
-    ocpdb-rabbitmq:
+    rabbitmq:
       condition: service_healthy
   volumes:
     - ./etc/ocpdb/config.yaml:/app/config.yaml
@@ -40,7 +40,7 @@ x-park-api-defaults: &park-api-defaults
   depends_on:
     park-api-db:
       condition: service_healthy
-    park-api-rabbitmq:
+    rabbitmq:
       condition: service_healthy
   volumes:
     - ./etc/park-api/config.yaml:/app/config.yaml
@@ -49,20 +49,6 @@ x-park-api-defaults: &park-api-defaults
     - ./var/park-api/logs:/app/logs
     - ./var/park-api/temp:/app/temp
 
-x-generic-rabbitmq: &generic-rabbitmq
-  networks: [ipl]
-  image: rabbitmq:3.13
-  user: rabbitmq  # required due eacces-issue: https://github.com/docker-library/rabbitmq/issues/318
-  environment:
-    # Disable spammy logging
-    RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: '-rabbit log [{console,[{level,warning}]}]'
-  healthcheck:
-    test: rabbitmq-diagnostics -q ping
-    interval: 3s
-    timeout: 1s
-    retries: 20
-  restart: unless-stopped
-
 services:
   # Traefik is our ingress: requests enter our infrastructure through it.
   # It is an HTTP reverse proxy, discovering available services (and their containers)
@@ -204,10 +190,10 @@ services:
     restart: unless-stopped
     healthcheck:
       test: ps aux | grep -q 'x2gbfs\.x2gbfs' || exit 1
-      # If you want to wait until all feeds have been generated 
-      # before depending lamassu starts subscribing them, 
+      # If you want to wait until all feeds have been generated
+      # before depending lamassu starts subscribing them,
       # configure an appropriate startup_period, e.g. 60s
-      # Otherwise, lamassu might update them only after the 
+      # Otherwise, lamassu might update them only after the
       # update interval period has passed, which for test might
       # last several hours.
       start_interval: ${X2GBFS_HEALTHCHECK_START_INTERVAL:-1s}
@@ -711,6 +697,24 @@ services:
       - "traefik.http.services.gtfs-api-docs.loadbalancer.server.port=8080"
     restart: unless-stopped
 
+  rabbitmq:
+    networks: [ipl]
+    image: ${RABBITMQ_VERSION}
+    user: rabbitmq  # required due eacces-issue: https://github.com/docker-library/rabbitmq/issues/318
+    environment:
+      # Disable spammy logging
+      RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: '-rabbit log [{console,[{level,warning}]}]'
+    healthcheck:
+      test: rabbitmq-diagnostics -q ping
+      interval: 30s
+      start_period: 20s
+      start_interval: 5s
+      timeout: 10s
+    volumes:
+      - ./etc/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf
+      - ./etc/rabbitmq/definitions.json:/etc/rabbitmq/definitions.json
+    restart: unless-stopped
+
   ocpdb-flask:
     <<: *ocpdb-defaults
     command: ["gunicorn", "--bind", "0.0.0.0:5000", "webapp.entry_point_gunicorn:app"]
@@ -729,9 +733,14 @@ services:
     restart: unless-stopped
     command: ["celery", "-A", "webapp.entry_point_celery", "worker", "-c", "2"]
 
+  ocpdb-heartbeat:
+    <<: *ocpdb-defaults
+    restart: unless-stopped
+    command: ["celery", "-A", "webapp.entry_point_celery", "beat", "-s", "/tmp/celerybeat-schedule"]
+
   ocpdb-init:
     <<: *ocpdb-defaults
-    command: ["sh", "-c", "flask db upgrade && flask bnetza import-web"]
+    command: ["sh", "-c", "flask db upgrade && flask source fetch-all"]
 
   ocpdb-db:
     networks: [ipl]
@@ -749,9 +758,6 @@ services:
       retries: 3
     restart: unless-stopped
 
-  ocpdb-rabbitmq:
-    <<: *generic-rabbitmq
-
   park-api-flask:
     <<: *park-api-defaults
     restart: unless-stopped
@@ -795,9 +801,6 @@ services:
       retries: 3
     restart: unless-stopped
 
-  park-api-rabbitmq:
-    <<: *generic-rabbitmq
-
   sftp:
     image: ${SFTP_IMAGE}
     networks: [ipl]

diff --git a/etc/ocpdb/config.yaml b/etc/ocpdb/config.yaml
@@ -10,4 +10,9 @@ OPENAPI_SERVERS:
 # So far, we don't have credentials, so we can put REMOTE_SERVERS here
 REMOTE_SERVERS:
   BNETZA:
-    url: https://data.bundesnetzagentur.de/Bundesnetzagentur/SharedDocs/Downloads/DE/Sachgebiete/Energie/Unternehmen_Institutionen/E_Mobilitaet/Ladesaeulenregister.xlsx
+    url: https://data.bundesnetzagentur.de/Bundesnetzagentur/SharedDocs/Downloads/DE/Sachgebiete/Energie/Unternehmen_Institutionen/E_Mobilitaet/Ladesaeulenregister_01122024.xlsx
+  SW_STUTTGART:
+    url: 'https://new-poi.chargecloud.de'
+
+AUTO_FETCH_SOURCES:
+  - sw_stuttgart
diff --git a/etc/rabbitmq/definitions.json b/etc/rabbitmq/definitions.json
@@ -0,0 +1,57 @@
+{
+    "permissions": [
+        {
+            "configure": ".*",
+            "read": ".*",
+            "user": "guest",
+            "vhost": "/",
+            "write": ".*"
+        },
+        {
+            "configure": ".*",
+            "read": ".*",
+            "user": "guest",
+            "vhost": "ocpdb",
+            "write": ".*"
+        },
+        {
+            "configure": ".*",
+            "read": ".*",
+            "user": "guest",
+            "vhost": "park-api",
+            "write": ".*"
+        }
+    ],
+    "rabbit_version": "3.13.7",
+    "rabbitmq_version": "3.13.7",
+    "users": [
+        {
+            "hashing_algorithm": "rabbit_password_hashing_sha256",
+            "name": "guest",
+            "password_hash": "jJ6i6IgumvqlsjiFXbmB57Kr1K4fXCsNVk+Wrw4r9Rx+/6kq",
+            "tags": [
+                "administrator"
+            ]
+        }
+    ],
+    "vhosts": [
+        {
+            "metadata": {
+                "description": "Default virtual host"
+            },
+            "name": "/"
+        },
+        {
+            "metadata": {
+                "description": "Virtual host for ParkAPI"
+            },
+            "name": "park-api"
+        },
+        {
+            "metadata": {
+                "description": "Virtual host for OCPDB"
+            },
+            "name": "ocpdb"
+        }
+    ]
+}
diff --git a/etc/rabbitmq/rabbitmq.conf b/etc/rabbitmq/rabbitmq.conf
@@ -0,0 +1,5 @@
+# Just reload definitions if they are changed
+definitions.skip_if_unchanged = true
+# Point to definitions file
+definitions.import_backend = local_filesystem
+definitions.local.path = /etc/rabbitmq/definitions.json