Skip to content

Commit

Permalink
Merge branch 'develop' into rename_master_branch
Browse files Browse the repository at this point in the history
  • Loading branch information
xmatthias committed Sep 21, 2020
2 parents 637fe35 + 87f6c7b commit d639290
Show file tree
Hide file tree
Showing 22 changed files with 206 additions and 88 deletions.
18 changes: 18 additions & 0 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
FROM freqtradeorg/freqtrade:develop

# Install dependencies
COPY requirements-dev.txt /freqtrade/
RUN apt-get update \
&& apt-get -y install git sudo vim \
&& apt-get clean \
&& pip install autopep8 -r docs/requirements-docs.txt -r requirements-dev.txt --no-cache-dir \
&& useradd -u 1000 -U -m ftuser \
&& mkdir -p /home/ftuser/.vscode-server /home/ftuser/.vscode-server-insiders /home/ftuser/commandhistory \
&& echo "export PROMPT_COMMAND='history -a'" >> /home/ftuser/.bashrc \
&& echo "export HISTFILE=~/commandhistory/.bash_history" >> /home/ftuser/.bashrc \
&& chown ftuser: -R /home/ftuser/

USER ftuser

# Empty the ENTRYPOINT to allow all commands
ENTRYPOINT []
44 changes: 44 additions & 0 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"name": "freqtrade Develop",

"dockerComposeFile": [
"docker-compose.yml"
],

"service": "ft_vscode",

"workspaceFolder": "/freqtrade/",

"settings": {
"terminal.integrated.shell.linux": "/bin/bash",
"editor.insertSpaces": true,
"files.trimTrailingWhitespace": true,
"[markdown]": {
"files.trimTrailingWhitespace": false,
},
"python.pythonPath": "/usr/local/bin/python",
},

// Add the IDs of extensions you want installed when the container is created.
"extensions": [
"ms-python.python",
"ms-python.vscode-pylance",
"davidanson.vscode-markdownlint",
"ms-azuretools.vscode-docker",
],

// Use 'forwardPorts' to make a list of ports inside the container available locally.
// "forwardPorts": [],

// Uncomment the next line if you want start specific services in your Docker Compose config.
// "runServices": [],

// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
// "shutdownAction": "none",

// Uncomment the next line to run commands after the container is created - for example installing curl.
// "postCreateCommand": "sudo apt-get update && apt-get install -y git",

// Uncomment to connect as a non-root user if you've added one. See https://aka.ms/vscode-remote/containers/non-root.
"remoteUser": "ftuser"
}
24 changes: 24 additions & 0 deletions .devcontainer/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
---
version: '3'
services:
ft_vscode:
build:
context: ..
dockerfile: ".devcontainer/Dockerfile"
volumes:
# Allow git usage within container
- "/home/${USER}/.ssh:/home/ftuser/.ssh:ro"
- "/home/${USER}/.gitconfig:/home/ftuser/.gitconfig:ro"
- ..:/freqtrade:cached
# Persist bash-history
- freqtrade-vscode-server:/home/ftuser/.vscode-server
- freqtrade-bashhistory:/home/ftuser/commandhistory
# Expose API port
ports:
- "127.0.0.1:8080:8080"
command: /bin/sh -c "while sleep 1000; do :; done"


volumes:
freqtrade-vscode-server:
freqtrade-bashhistory:
1 change: 1 addition & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ CONTRIBUTING.md
MANIFEST.in
README.md
freqtrade.service
user_data
3 changes: 2 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ RUN pip install numpy --no-cache-dir \

# Install and execute
COPY . /freqtrade/
RUN pip install -e . --no-cache-dir
RUN pip install -e . --no-cache-dir \
&& mkdir /freqtrade/user_data/
ENTRYPOINT ["freqtrade"]
# Default to trade mode
CMD [ "trade" ]
3 changes: 3 additions & 0 deletions build_helpers/publish_docker.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

# Replace / with _ to create a valid tag
TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g")
TAG_PLOT=${TAG}_plot
echo "Running for ${TAG}"

# Add commit and commit_message to docker container
Expand All @@ -16,6 +17,7 @@ else
docker pull ${IMAGE_NAME}:${TAG}
docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} .
fi
docker build --cache-from freqtrade:${TAG} --build-arg sourceimage=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot .

if [ $? -ne 0 ]; then
echo "failed building image"
Expand All @@ -32,6 +34,7 @@ fi

# Tag image for upload
docker tag freqtrade:$TAG ${IMAGE_NAME}:$TAG
docker tag freqtrade:$TAG_PLOT ${IMAGE_NAME}:$TAG_PLOT
if [ $? -ne 0 ]; then
echo "failed tagging image"
return 1
Expand Down
20 changes: 0 additions & 20 deletions docker-compose.develop.yml

This file was deleted.

2 changes: 2 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ services:
freqtrade:
image: freqtradeorg/freqtrade:stable
# image: freqtradeorg/freqtrade:develop
# Use plotting image
# image: freqtradeorg/freqtrade:develop_plot
# Build step - only needed when additional dependencies are needed
# build:
# context: .
Expand Down
1 change: 1 addition & 0 deletions Dockerfile.develop → docker/Dockerfile.develop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ FROM freqtradeorg/freqtrade:develop

# Install dependencies
COPY requirements-dev.txt /freqtrade/

RUN pip install numpy --no-cache-dir \
&& pip install -r requirements-dev.txt --no-cache-dir

Expand Down
7 changes: 7 additions & 0 deletions docker/Dockerfile.jupyter
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM freqtradeorg/freqtrade:develop_plot


RUN pip install jupyterlab --no-cache-dir

# Empty the ENTRYPOINT to allow all commands
ENTRYPOINT []
10 changes: 10 additions & 0 deletions docker/Dockerfile.plot
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
ARG sourceimage=develop
FROM freqtradeorg/freqtrade:${sourceimage}

# Install dependencies
COPY requirements-plot.txt /freqtrade/

RUN pip install -r requirements-plot.txt --no-cache-dir

# Empty the ENTRYPOINT to allow all commands
ENTRYPOINT []
File renamed without changes.
16 changes: 16 additions & 0 deletions docker/docker-compose-jupyter.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
---
version: '3'
services:
ft_jupyterlab:
build:
context: ..
dockerfile: docker/Dockerfile.jupyter
restart: unless-stopped
container_name: freqtrade
ports:
- "127.0.0.1:8888:8888"
volumes:
- "./user_data:/freqtrade/user_data"
# Default command used when running `docker compose up`
command: >
jupyter lab --port=8888 --ip 0.0.0.0 --allow-root
22 changes: 15 additions & 7 deletions docs/data-analysis.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,22 @@
# Analyzing bot data with Jupyter notebooks

You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/`.
You can analyze the results of backtests and trading history easily using Jupyter notebooks. Sample notebooks are located at `user_data/notebooks/` after initializing the user directory with `freqtrade create-userdir --userdir user_data`.

## Pro tips
## Quick start with docker

Freqtrade provides a docker-compose file which starts up a jupyter lab server.
You can run this server using the following command: `docker-compose -f docker/docker-compose-jupyter.yml up`

This will create a dockercontainer running jupyter lab, which will be accessible using `https://127.0.0.1:8888/lab`.
Please use the link that's printed in the console after startup for simplified login.

For more information, Please visit the [Data analysis with Docker](docker_quickstart.md#data-analayis-using-docker-compose) section.

### Pro tips

* See [jupyter.org](https://jupyter.org/documentation) for usage instructions.
* Don't forget to start a Jupyter notebook server from within your conda or venv environment or use [nb_conda_kernels](https://github.com/Anaconda-Platform/nb_conda_kernels)*
* Copy the example notebook before use so your changes don't get clobbered with the next freqtrade update.
* Copy the example notebook before use so your changes don't get overwritten with the next freqtrade update.

### Using virtual environment with system-wide Jupyter installation

Expand All @@ -28,10 +38,8 @@ ipython kernel install --user --name=freqtrade
!!! Note
This section is provided for completeness, the Freqtrade Team won't provide full support for problems with this setup and will recommend to install Jupyter in the virtual environment directly, as that is the easiest way to get jupyter notebooks up and running. For help with this setup please refer to the [Project Jupyter](https://jupyter.org/) [documentation](https://jupyter.org/documentation) or [help channels](https://jupyter.org/community).


## Fine print

Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually.
!!! Warning
Some tasks don't work especially well in notebooks. For example, anything using asynchronous execution is a problem for Jupyter. Also, freqtrade's primary entry point is the shell cli, so using pure python in a notebook bypasses arguments that provide required objects and parameters to helper functions. You may need to set those values or create expected objects manually.

## Recommended workflow

Expand Down
13 changes: 9 additions & 4 deletions docs/data-download.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ usage: freqtrade download-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[-p PAIRS [PAIRS ...]] [--pairs-file FILE]
[--days INT] [--timerange TIMERANGE]
[--dl-trades] [--exchange EXCHANGE]
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
[--erase]
[--data-format-ohlcv {json,jsongz,hdf5}]
[--data-format-trades {json,jsongz,hdf5}]
Expand All @@ -41,7 +41,7 @@ optional arguments:
as --timeframes/-t.
--exchange EXCHANGE Exchange name (default: `bittrex`). Only valid if no
config is provided.
-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]
-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
Specify which tickers to download. Space-separated
list. Default: `1m 5m`.
--erase Clean all existing data for the selected
Expand Down Expand Up @@ -71,6 +71,11 @@ Common arguments:
```

!!! Note "Startup period"
`download-data` is a strategy-independent command. The idea is to download a big chunk of data once, and then iteratively increase the amount of data stored.

For that reason, `download-data` does not care about the "startup-period" defined in a strategy. It's up to the user to download additional days if the backtest should start at a specific point in time (while respecting startup period).

### Data format

Freqtrade currently supports 3 data-formats for both OHLCV and trades data:
Expand Down Expand Up @@ -104,7 +109,7 @@ usage: freqtrade convert-data [-h] [-v] [--logfile FILE] [-V] [-c PATH]
[-p PAIRS [PAIRS ...]] --format-from
{json,jsongz,hdf5} --format-to
{json,jsongz,hdf5} [--erase]
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]]
[-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]]
optional arguments:
-h, --help show this help message and exit
Expand All @@ -117,7 +122,7 @@ optional arguments:
Destination format for data conversion.
--erase Clean all existing data for the selected
exchange/pairs/timeframes.
-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w} ...]
-t {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...], --timeframes {1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} [{1m,3m,5m,15m,30m,1h,2h,4h,6h,8h,12h,1d,3d,1w,2w,1M,1y} ...]
Specify which tickers to download. Space-separated
list. Default: `1m 5m`.
Expand Down
63 changes: 16 additions & 47 deletions docs/developer.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,24 @@ This will spin up a local server (usually on port 8000) so you can see if everyt

## Developer setup

To configure a development environment, best use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
Alternatively (if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.
To configure a development environment, you can either use the provided [DevContainer](#devcontainer-setup), or use the `setup.sh` script and answer "y" when asked "Do you want to install dependencies for dev [y/N]? ".
Alternatively (e.g. if your system is not supported by the setup.sh script), follow the manual installation process and run `pip3 install -e .[all]`.

This will install all required tools for development, including `pytest`, `flake8`, `mypy`, and `coveralls`.

### Devcontainer setup

The fastest and easiest way to get started is to use [VSCode](https://code.visualstudio.com/) with the Remote container extension.
This gives developers the ability to start the bot with all required dependencies *without* needing to install any freqtrade specific dependencies on your local machine.

#### Devcontainer dependencies

* [VSCode](https://code.visualstudio.com/)
* [docker](https://docs.docker.com/install/)
* [Remote container extension documentation](https://code.visualstudio.com/docs/remote)

For more information about the [Remote container extension](https://code.visualstudio.com/docs/remote), best consult the documentation.

### Tests

New code should be covered by basic unittests. Depending on the complexity of the feature, Reviewers may request more in-depth unittests.
Expand All @@ -50,51 +63,6 @@ def test_method_to_test(caplog):

```

### Local docker usage

The fastest and easiest way to start up is to use docker-compose.develop which gives developers the ability to start the bot up with all the required dependencies, *without* needing to install any freqtrade specific dependencies on your local machine.

#### Install

* [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
* [docker](https://docs.docker.com/install/)
* [docker-compose](https://docs.docker.com/compose/install/)

#### Starting the bot

##### Use the develop dockerfile

``` bash
rm docker-compose.yml && mv docker-compose.develop.yml docker-compose.yml
```

#### Docker Compose

##### Starting

``` bash
docker-compose up
```

![Docker compose up](https://user-images.githubusercontent.com/419355/65456322-47f63a80-de06-11e9-90c6-3c74d1bad0b8.png)

##### Rebuilding

``` bash
docker-compose build
```

##### Executing (effectively SSH into the container)

The `exec` command requires that the container already be running, if you want to start it
that can be effected by `docker-compose up` or `docker-compose run freqtrade_develop`

``` bash
docker-compose exec freqtrade_develop /bin/bash
```

![image](https://user-images.githubusercontent.com/419355/65456522-ba671a80-de06-11e9-9598-df9ca0d8dcac.png)

## ErrorHandling

Freqtrade Exceptions all inherit from `FreqtradeException`.
Expand Down Expand Up @@ -264,6 +232,7 @@ This documents some decisions taken for the CI Pipeline.

* CI runs on all OS variants, Linux (ubuntu), macOS and Windows.
* Docker images are build for the branches `stable` and `develop`.
* Docker images containing Plot dependencies are also available as `stable_plot` and `develop_plot`.
* Raspberry PI Docker images are postfixed with `_pi` - so tags will be `:stable_pi` and `develop_pi`.
* Docker images contain a file, `/freqtrade/freqtrade_commit` containing the commit this image is based of.
* Full docker image rebuilds are run once a week via schedule.
Expand Down
8 changes: 4 additions & 4 deletions docs/docker.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,16 +70,16 @@ cp -n config.json.example config.json

Best start by pulling the official docker image from dockerhub as explained [here](#download-the-official-docker-image) to speed up building.

To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image.
To add additional libraries to your docker image, best check out [Dockerfile.technical](https://github.com/freqtrade/freqtrade/blob/develop/docker/Dockerfile.technical) which adds the [technical](https://github.com/freqtrade/technical) module to the image.

```bash
docker build -t freqtrade -f Dockerfile.technical .
docker build -t freqtrade -f docker/Dockerfile.technical .
```

If you are developing using Docker, use `Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies:
If you are developing using Docker, use `docker/Dockerfile.develop` to build a dev Docker image, which will also set up develop dependencies:

```bash
docker build -f Dockerfile.develop -t freqtrade-dev .
docker build -f docker/Dockerfile.develop -t freqtrade-dev .
```

!!! Warning "Include your config file manually"
Expand Down
Loading

0 comments on commit d639290

Please sign in to comment.