Skip to content

Commit

Permalink
build(docker): updated docker trainer scripts
Browse files Browse the repository at this point in the history
oroszgy committed Oct 28, 2024
1 parent 8b2ccda commit e8cd213
Showing 3 changed files with 18 additions and 5 deletions.
6 changes: 6 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -41,6 +41,8 @@ RUN add-apt-repository ppa:deadsnakes/ppa

RUN apt-get update && apt install -y \
python3.12 \
python3.12-dev \
python3.12-venv \
python3.12-distutils \
git \
&& rm -rf /var/lib/apt/lists/*
@@ -58,6 +60,10 @@ COPY poetry.toml /app/model

RUN poetry install

# Workaround for having setuptools available
RUN poetry run python -m ensurepip --upgrade
RUN poetry run python -m pip install --upgrade setuptools

COPY meta.json /app/model
COPY project.yml /app/model
COPY --from=root train.sh /app/model
10 changes: 8 additions & 2 deletions train.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,10 @@
#!/bin/bash

poetry run spacy project assets
GPU=0 poetry run spacy project run all
if [ -z "$1" ]; then
echo "Running end-to-end training"
poetry run spacy project assets
GPU=0 poetry run spacy project run all
else
echo "Running '$1'"
GPU=0 poetry run spacy project run $1
fi
7 changes: 4 additions & 3 deletions train_w_docker.sh
Original file line number Diff line number Diff line change
@@ -12,9 +12,10 @@ if [ ! -d "$1" ]; then
fi

model_dir=$1
model_name=$(basename $model_dir)

docker buildx build -t trainer-${model_dir} "${model_dir}" --build-context root="." -f ./Dockerfile
docker run -it --rm --name trainer-${model_dir} \
docker buildx build -t trainer-${model_name} "${model_dir}" --build-context root="." -f ./Dockerfile
docker run -it --rm --name trainer-${model_name} \
--runtime=nvidia \
-v "$(pwd)/${model_dir}"/data:/app/model/data \
-v "$(pwd)/${model_dir}"/models:/app/model/models \
@@ -23,4 +24,4 @@ docker run -it --rm --name trainer-${model_dir} \
-v "$(pwd)/${model_dir}"/wandb:/app/model/wandb \
-v "$(pwd)/${model_dir}"/../huspacy:/app/huspacy \
-v "$(pwd)/${model_dir}"/../scripts:/app/scripts \
trainer-${model_dir} "./train.sh"
trainer-${model_name} "./train.sh" "${@:2}"

0 comments on commit e8cd213

Please sign in to comment.