Initial commit

tanega · Jun 13, 2023 · 7283c87 · 7283c87
commit 7283c87
Show file tree

Hide file tree

Showing 16 changed files with 2,857 additions and 0 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,6 @@
+.gitignore
+.env*
+.vscode/
+.pytest_cache/
+data/
+dist/
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,95 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# DotEnv configuration
+.env
+
+# Database
+*.db
+*.rdb
+
+# Pycharm
+.idea
+
+# VS Code
+.vscode/
+
+# Spyder
+.spyproject/
+
+# Jupyter NB Checkpoints
+.ipynb_checkpoints/
+
+# exclude data from source control by default
+/data*/
+
+# Mac OS-specific storage files
+.DS_Store
+
+# vim
+*.swp
+*.swo
+
+# Mypy cache
+.mypy_cache/
+
+# Pytest cache
+.pytest_cache/
+
+# ignore any env
+.env*
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,100 @@
+###########################################
+# dev
+###########################################
+ARG BASE_IMAGE=python:3.9.8-slim-buster
+FROM "${BASE_IMAGE}" as dev
+
+ENV PYTHONUNBUFFERED=True
+ENV PIP_NO_CACHE_DIR=True
+
+# Install the python package managers.
+RUN pip install -U \
+    pip \
+    setuptools \
+    wheel \
+    poetry
+
+# Set this folder at the system root and then cd into it.
+WORKDIR /usr/src/app
+
+# Copy poetry's package list and then install all non-developmental dependencies.
+COPY poetry.lock pyproject.toml ./
+
+# Installs dependencies, but does *not* install our python code as a package because it's not mounted. Must be added afterward!
+ENV POETRY_VIRTUALENVS_IN_PROJECT=false
+RUN poetry install 
+
+# Setup a local instance of dagit
+RUN mkdir -p /opt/dagster/dagster_home
+ENV DAGSTER_HOME=/opt/dagster/dagster_home/
+
+# Using a "heredoc" called `dev_command.sh` because this is the only place and time this script will be needed
+RUN echo "poetry install" > /usr/bin/dev_command.sh
+RUN echo "poetry run dagster-daemon run &" >> /usr/bin/dev_command.sh
+RUN echo "poetry run dagit -h 0.0.0.0 -p 3000" >> /usr/bin/dev_command.sh
+RUN chmod +x /usr/bin/dev_command.sh
+
+EXPOSE 3000
+CMD ["bash", "dev_command.sh"]
+
+###########################################
+# build
+###########################################
+FROM dev as build
+
+WORKDIR /usr/src/app
+
+# Removes dev dependencies (reuses installed packages from previous docker stage)
+RUN poetry install --no-dev
+
+# Copy the contents of the project root directory to the app directory in the container
+COPY . .
+
+# Builds our code and dependencies into a python wheel, then saves the resulting name to `package_name`
+RUN poetry build --format wheel | grep "Built" | sed 's/^.*\s\(.*\.whl\)/\1/' > package_name
+
+# Override the CMD because we don't really need to run anything here
+CMD ["python3"]
+###########################################
+# deploy
+###########################################
+
+FROM "${BASE_IMAGE}"
+ARG DAGSTER_VERSION=0.13.19
+
+# ==> Add Dagster layer
+RUN \
+# Cron
+       apt-get update -yqq \
+    && apt-get install -yqq cron \
+# Dagster
+    && pip install -U --no-cache-dir \
+        pip \
+        setuptools \
+        wheel \
+        dagster==${DAGSTER_VERSION} \
+        dagster-postgres==${DAGSTER_VERSION} \
+        dagster-celery[flower,redis,kubernetes]==${DAGSTER_VERSION} \
+        dagster-aws==${DAGSTER_VERSION} \
+        dagster-k8s==${DAGSTER_VERSION} \
+        dagster-celery-k8s==${DAGSTER_VERSION} \
+# Cleanup
+# "| :" forces a success signal even if some cleanup fails
+    &&  rm -rf /var | : \
+    &&  rm -rf /root/.cache  \
+    &&  rm -rf /usr/lib/python2.7 \
+    &&  rm -rf /usr/lib/x86_64-linux-gnu/guile
+
+# ==> Add user code layer
+WORKDIR /usr/src/app
+
+# Copy our python package (wheel file, output of `poetry build`) and install it
+COPY --from=build /usr/src/app/dist repo_package
+COPY --from=build /usr/src/app/package_name package_name
+RUN pip install --no-cache-dir repo_package/$(cat package_name)
+
+# Defines dagster repo(s)
+COPY workspace.yaml workspace.yaml
+
+# Default Job Configs
+COPY job_configs job_configs