[wip] dagster multi containers

tanega · Jun 21, 2023 · f40a5a6 · f40a5a6
1 parent 1ba362b
commit f40a5a6
Show file tree

Hide file tree

Showing 10 changed files with 252 additions and 1 deletion.
diff --git a/dagster_multi/Dockerfile_dagster b/dagster_multi/Dockerfile_dagster
@@ -0,0 +1,20 @@
+# Dagster libraries to run both dagit and the dagster-daemon. Does not
+# need to have access to any pipeline code.
+
+FROM python:3.7-slim
+
+RUN pip install \
+    dagster \
+    dagster-graphql \
+    dagit \
+    dagster-postgres \
+    dagster-docker
+
+# Set $DAGSTER_HOME and copy dagster instance and workspace YAML there
+ENV DAGSTER_HOME=/opt/dagster/dagster_home/
+
+RUN mkdir -p $DAGSTER_HOME
+
+COPY dagster.yaml workspace.yaml $DAGSTER_HOME
+
+WORKDIR $DAGSTER_HOME
diff --git a/dagster_multi/Dockerfile_user_code b/dagster_multi/Dockerfile_user_code
@@ -0,0 +1,23 @@
+FROM python:3.7-slim
+
+# Checkout and install dagster libraries needed to run the gRPC server
+# exposing your repository to dagit and dagster-daemon, and to load the DagsterInstance
+
+RUN pip install \
+    dagster \
+    dagster-postgres \
+    dagster-docker
+
+# Add repository code
+
+WORKDIR /opt/dagster/app
+
+COPY repo.py /opt/dagster/app
+
+# Run dagster gRPC server on port 4000
+
+EXPOSE 4000
+
+# CMD allows this to be overridden from run launchers or executors that want
+# to run other commands against your repository
+CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-f", "repo.py"]
diff --git a/dagster_multi/__init__.py b/dagster_multi/__init__.py
diff --git a/dagster_multi/dagster.yaml b/dagster_multi/dagster.yaml
@@ -0,0 +1,63 @@
+scheduler:
+  module: dagster.core.scheduler
+  class: DagsterDaemonScheduler
+
+run_coordinator:
+  module: dagster.core.run_coordinator
+  class: QueuedRunCoordinator
+
+run_launcher:
+  module: dagster_docker
+  class: DockerRunLauncher
+  config:
+    env_vars:
+      - DAGSTER_POSTGRES_USER
+      - DAGSTER_POSTGRES_PASSWORD
+      - DAGSTER_POSTGRES_DB
+    network: docker_example_network
+    container_kwargs:
+      volumes: # Make docker client accessible to any launched containers as well
+        - /var/run/docker.sock:/var/run/docker.sock
+        - /tmp/io_manager_storage:/tmp/io_manager_storage
+
+run_storage:
+  module: dagster_postgres.run_storage
+  class: PostgresRunStorage
+  config:
+    postgres_db:
+      hostname: postgres
+      username:
+        env: DAGSTER_POSTGRES_USER
+      password:
+        env: DAGSTER_POSTGRES_PASSWORD
+      db_name:
+        env: DAGSTER_POSTGRES_DB
+      port: 5432
+
+schedule_storage:
+  module: dagster_postgres.schedule_storage
+  class: PostgresScheduleStorage
+  config:
+    postgres_db:
+      hostname: postgres
+      username:
+        env: DAGSTER_POSTGRES_USER
+      password:
+        env: DAGSTER_POSTGRES_PASSWORD
+      db_name:
+        env: DAGSTER_POSTGRES_DB
+      port: 5432
+
+event_log_storage:
+  module: dagster_postgres.event_log
+  class: PostgresEventLogStorage
+  config:
+    postgres_db:
+      hostname: postgres
+      username:
+        env: DAGSTER_POSTGRES_USER
+      password:
+        env: DAGSTER_POSTGRES_PASSWORD
+      db_name:
+        env: DAGSTER_POSTGRES_DB
+      port: 5432
diff --git a/dagster_multi/repo.py b/dagster_multi/repo.py
@@ -0,0 +1,40 @@
+from dagster import FilesystemIOManager, graph, op, repository, schedule
+from dagster_docker import docker_executor
+
+
+@op
+def hello():
+    return 1
+
+
+@op
+def goodbye(foo):
+    if foo != 1:
+        raise Exception("Bad io manager")
+    return foo * 2
+
+
+@graph
+def my_graph():
+    goodbye(hello())
+
+
+my_job = my_graph.to_job(name="my_job")
+
+my_step_isolated_job = my_graph.to_job(
+    name="my_step_isolated_job",
+    executor_def=docker_executor,
+    resource_defs={
+        "io_manager": FilesystemIOManager(base_dir="/tmp/io_manager_storage")
+    },
+)
+
+
+@schedule(cron_schedule="* * * * *", job=my_job, execution_timezone="US/Central")
+def my_schedule(_context):
+    return {}
+
+
+@repository
+def deploy_docker_repository():
+    return [my_job, my_step_isolated_job, my_schedule]
diff --git a/dagster_multi/requirements.txt b/dagster_multi/requirements.txt
@@ -0,0 +1,3 @@
+dagster
+dagit
+dagster-docker
diff --git a/dagster_multi/tox.ini b/dagster_multi/tox.ini
@@ -0,0 +1,14 @@
+[tox]
+envlist = py{39,38,37,36}-{unix,windows}
+skipsdist = True
+
+[testenv]
+download = True
+passenv = CI_* COVERALLS_REPO_TOKEN BUILDKITE* DEPLOY_DOCKER_DAGIT_HOST
+deps =
+  -e ../../python_modules/dagster[test]
+allowlist_externals =
+  /bin/bash
+commands =
+  !windows: /bin/bash -c '! pip list --exclude-editable | grep -e dagster -e dagit'
+  pytest -c ../../pyproject.toml -vv {posargs}
diff --git a/dagster_multi/workspace.yaml b/dagster_multi/workspace.yaml
@@ -0,0 +1,6 @@
+load_from:
+  # Each entry here corresponds to a service in the docker-compose file that exposes user code.
+  - grpc_server:
+      host: dagster_multi_user_code
+      port: 4000
+      location_name: "edagster_multi_user_code_1"
diff --git a/docker/.env b/docker/.env
@@ -12,9 +12,24 @@ TRAEFIK_DASHBOARD_PORT=8080
 DAGSTER_SUBDOMAIN=orchestration
 # SLACK_HOOK=https://hooks.slack.com/services/YOUR/WEBHOOK/URL
 
+#### DAGSTER_MULTI ####
+DAGSTER_MULTI_SUBDOMAIN=orch_multi
+# SLACK_HOOK=https://hooks.slack.com/services/YOUR/WEBHOOK/URL
+
+#### DAGSTER_MULTI-USER_CODE ####
+DAGSTER_POSTGRES_USER=user
+DAGSTER_POSTGRES_PASSWORD=password
+DAGSTER_POSTGRES_DB=dagster_storage
+DAGSTER_CURRENT_IMAGE=user_code_image
+
 #### METABASE/DUCK ####
 MB_PLUGINS_DIR=/home/plugins
 
 #### MINIO ####
 MINIO_ROOT_USER=admin
-MINIO_ROOT_PASSWORD=password
+MINIO_ROOT_PASSWORD=password
+
+#### POSTGRESQL ####
+POSTGRES_USER=user
+POSTGRES_PASSWORD=password
+POSTGRES_DB=dagster_storage
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -36,6 +36,73 @@ services:
       - "80:80"
       - "443:443"
       - "8080:8080"
+    # This service runs the postgres DB used by dagster for run storage, schedule storage,
+  # and event log storage.
+  postgres:
+    <<: *project_defaults
+    image: postgres:11
+    container_name: postgresql
+
+  # This service runs the gRPC server that loads your user code, in both dagit
+  # and dagster-daemon. By setting DAGSTER_CURRENT_IMAGE to its own image, we tell the
+  # run launcher to use this same image when launching runs in a new container as well.
+  # Multiple containers like this can be deployed separately - each just needs to run on
+  # its own port, and have its own entry in the workspace.yaml file that's loaded by dagit.
+  dagster_multi_user_code:
+    <<: *project_defaults
+    build:
+      context: ./dagster_multi
+      dockerfile: Dockerfile_user_code
+    container_name: user_code
+    image: user_code_image
+
+  # This service runs dagit, which loads your user code from the user code container.
+  # Since our instance uses the QueuedRunCoordinator, any runs submitted from dagit will be put on
+  # a queue and later dequeued and launched by dagster-daemon.
+  dagster_multi_dagit:
+    <<: *project_defaults
+    build:
+      context: ./dagster_multi
+      dockerfile: Dockerfile_dagster
+    entrypoint:
+      - dagit
+      - -h
+      - "0.0.0.0"
+      - -p
+      - "3000"
+      - -w
+      - workspace.yaml
+    container_name: dagster_multi_dagit
+    ports:
+      - "3030:3000"
+    volumes: # Make docker client accessible so we can terminate containers from dagit
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp/io_manager_storage:/tmp/io_manager_storage
+    depends_on:
+      - postgres
+      - dagster_multi_user_code
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.dagster-multi.rule=Host(`orch-multi.localhost`)"
+      - "traefik.http.routers.dagster-multi.entrypoints=web"
+
+  # This service runs the dagster-daemon process, which is responsible for taking runs
+  # off of the queue and launching them, as well as creating runs from schedules or sensors.
+  daemon:
+    <<: *project_defaults
+    build:
+      context: ./dagster_multi
+      dockerfile: Dockerfile_dagster
+    entrypoint:
+      - dagster-daemon
+      - run
+    container_name: daemon
+    volumes: # Make docker client accessible so we can launch containers using host docker
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp/io_manager_storage:/tmp/io_manager_storage
+    depends_on:
+      - postgres
+      - dagster_multi_user_code
 
   dagster:
     <<: *project_defaults