diff --git a/readthedocs.yml b/readthedocs.yml index efe2168d..95f7f4be 100644 --- a/readthedocs.yml +++ b/readthedocs.yml @@ -1,8 +1,6 @@ -# .readthedocs.yml - build: image: latest -# Use Python 3.6, specifically to support f-string. +# TODO: change to Python 3.7 once ReadtheDocs approves the PR python: version: 3.6 diff --git a/requirements.txt b/requirements.txt index 1e3c7a7f..6fb980a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,45 @@ -cloudpickle>=0.5.3 -cython>=0.28.4 -pyyaml>=3.12 -setuptools>=39.0.1 -numpy>=1.14.5 +# Update: 2018-10-11 + +# System-related +mkl>=2019.0 +mkl-include>=2019.0 +cmake>=3.12.0 +cython>=0.28.5 +cffi>=1.11.5 +typing>=3.6.6 +msgpack>=0.5.6 +setuptools>=40.4.3 +wheel>=0.32.1 +twine>=1.12.1 + +# Common +jupyterlab >= 0.35.0 +numpy>=1.15.2 scipy>=1.1.0 -matplotlib>=2.2.2 -scikit-image>=0.14.0 -scikit-learn>=0.19.2 -imageio>=2.3.0 +pandas>=0.23.4 +cloudpickle>=0.6.1 +pyyaml>=3.13 +colored>=1.3.93 + +# Testing, PEP8 code quality +pytest>=3.8.2 +flake8>=3.5.0 + +# Image, Video +scikit-image>=0.14.1 +scikit-learn>=0.20.0 +pillow>=5.3.0 +imageio>=2.4.1 pyglet>=1.3.2 -pandas>=0.23.3 -seaborn>=0.8.1 -jupyterlab >= 0.32.1 -gym>= 0.10.5 + +# Plotting +matplotlib>=3.0.0 +seaborn>=0.9.0 + +# RL +gym>= 0.10.8 cma>=2.6.0 -pytest>=3.6.3 -flake8>=3.5.0 -sphinx>=1.7.6 -sphinx_rtd_theme>=0.4.0 -colored>=1.3.93 + +# ReadtheDocs +sphinx>=1.8.1 +sphinx_rtd_theme>=0.4.2 diff --git a/scripts/0_install_nvidia_kit.sh b/scripts/0_install_nvidia_kit.sh index 72cb38d3..b497b3ed 100644 --- a/scripts/0_install_nvidia_kit.sh +++ b/scripts/0_install_nvidia_kit.sh @@ -3,24 +3,22 @@ ######################################## # Install NVIDIA Drivers/CUDA/cuDNN # -# Date: 2018-08-10 # +# Date: 2018-10-11 # # Author: Xingdong Zuo # ######################################## # Stop when some intermediate command is failed set -e -# Create list of variables -# Find corresponding Driver runfile from Advanced Driver Search (with Beta version) -# http://www.nvidia.com/Download/Find.aspx +# Download GPU Driver file: http://www.nvidia.com/Download/Find.aspx export DRIVER_URL=http://us.download.nvidia.com/XFree86/Linux-x86_64/396.51/NVIDIA-Linux-x86_64-396.51.run export DRIVER_RUNFILE=NVIDIA-Linux-x86_64-396.51.run -# Find CUDA runfile from NVIDIA website -# https://developer.nvidia.com/cuda-downloads +export DRIVER_VERSION=396 +# Download CUDA runfile: https://developer.nvidia.com/cuda-downloads export CUDA_URL=https://developer.nvidia.com/compute/cuda/9.2/Prod2/local_installers/cuda_9.2.148_396.37_linux export CUDA_RUNFILE=cuda_9.2.148_396.37_linux export CUDA_VERSION=9.2 -# cuDNN file: Download from https://developer.nvidia.com/rdp/cudnn-download +# Download cuDNN file: https://developer.nvidia.com/rdp/cudnn-download export CUDNN_FILE=cudnn-9.2-linux-x64-v7.2.1.38.tgz # Disable nouveau driver: otherwise distribution provided pre-install will fail @@ -60,6 +58,13 @@ chmod +x ./$DRIVER_RUNFILE # make it executable sudo ./$DRIVER_RUNFILE --dkms --no-opengl-files # no OpenGL to support xvfb for fake screen on server echo 'NVIDIA driver installation is completed !' +# Append environment variables to .bashrc +echo "# Appended by Nvidia Driver installer" >> ~/.bashrc +echo "export PATH=\$PATH:/usr/lib/nvidia-$DRIVER_VERSION/bin/" >> ~/.bashrc +echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/lib/nvidia-$DRIVER_VERSION/" >> ~/.bashrc +echo "" >> ~/.bashrc +. ~/.bashrc # refresh bashrc + # Remove driver runfile rm ./$DRIVER_RUNFILE diff --git a/scripts/2_install_Anaconda.sh b/scripts/2_install_Anaconda.sh index 9185b3d9..80bbd942 100755 --- a/scripts/2_install_Anaconda.sh +++ b/scripts/2_install_Anaconda.sh @@ -2,7 +2,7 @@ ############################# # Install Anaconda with env # -# Date: 2018-09-07 # +# Date: 2018-10-11 # # Author: Xingdong Zuo # ############################# @@ -10,8 +10,8 @@ set -e # Create list of variables -export ANACONDA=Anaconda3-5.2.0-Linux-x86_64.sh # Check new version -export ENV_NAME=RL # name of Anaconda environment +export ANACONDA=Anaconda3-5.3.0-Linux-x86_64.sh # Check new version +export ENV_NAME=lagom # name of Anaconda environment export PYTHON_VERSION=3.7 # Python version # Download and install Anaconda @@ -21,10 +21,6 @@ chmod +x ./$ANACONDA # make it executable ./$ANACONDA # New version: don't choose yes to append PATH variable rm ./$ANACONDA # remove the installation file -# Append to load conda command from next run -# Remove PATH part in bashrc -echo ". $HOME/anaconda3/etc/profile.d/conda.sh" >> ~/.bashrc - # Alias for conda command export CONDA=$HOME/anaconda3/bin/conda @@ -34,11 +30,17 @@ $CONDA update -n base conda # Create an Anaconda environment with latest Python $CONDA create -n $ENV_NAME python=$PYTHON_VERSION -# Append to bashrc to automatically load virtual environment for each start -echo "conda activate $ENV_NAME # automatically activate virtual envs" >> ~/.bashrc # double quotation works for variable +# Append environment variables to .bashrc +echo "# Appended by Anaconda installer" >> ~/.bashrc +echo ". $HOME/anaconda3/etc/profile.d/conda.sh" >> ~/.bashrc # Load conda command +echo "export LIBRARY_PATH=\$LIBRARY_PATH:$HOME/anaconda3/envs/$ENV_NAME/lib" >> ~/.bashrc +echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:$HOME/anaconda3/envs/$ENV_NAME/lib" >> ~/.bashrc +echo "export PKG_CONFIG_PATH=\$PKG_CONFIG_PATH:$HOME/anaconda3/envs/$ENV_NAME/lib/pkgconfig/" >> ~/.bashrc +echo "conda activate $ENV_NAME # automatically activate virtual envs" >> ~/.bashrc # auto-load environment +echo "" >> ~/.bashrc +. ~/.bashrc # refresh bashrc # Message to restart the shell -echo "############################" +echo "#####################################################################" echo "Please restart the shell to automatically load conda and environment." echo "Then run the second bash file to set up dependencies." - diff --git a/scripts/3_setup_envs.sh b/scripts/3_setup_envs.sh index df1582d3..72fb712b 100755 --- a/scripts/3_setup_envs.sh +++ b/scripts/3_setup_envs.sh @@ -2,64 +2,24 @@ ############################# # Install dependencies # -# Date: 2018-08-10 # +# Date: 2018-10-11 # # Author: Xingdong Zuo # ############################# # Stop when some intermediate command is failed set -e -# Create list of variables -export ENV_NAME=RL # name of Anaconda environment -export NVIDIA_DRIVER_VER=396 # major version of Nvidia driver - # Alias of conda command export CONDA=$HOME/anaconda3/bin/conda -# Update all installed packages -$CONDA update --all - -# Append more useful things to bashrc -echo "# PATH" >> ~/.bashrc -echo "export PATH=/usr/lib/nvidia-$NVIDIA_DRIVER_VER/bin/:\$PATH" >> ~/.bashrc - -echo "# LIBRARY" >> ~/.bashrc -echo "export LIBRARY_PATH=$HOME/anaconda3/envs/$ENV_NAME/lib:\$LIBRARY_PATH" >> ~/.bashrc -echo "export LD_LIBRARY_PATH=/usr/lib/nvidia-$NVIDIA_DRIVER_VER:$HOME/anaconda3/envs/$ENV_NAME/lib:\$LD_LIBRARY_PATH" >> ~/.bashrc -echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:$HOME/.mujoco/mjpro150/bin" >> ~/.bashrc - -echo "# PKG_CONFIG" >> ~/.bashrc -echo "export PKG_CONFIG_PATH=\$PKG_CONFIG_PATH:$HOME/anaconda3/envs/$ENV_NAME/lib/pkgconfig/" >> ~/.bashrc - # Set up .vimrc for tab as 4 spaces echo "set tabstop=4" >> ~/.vimrc echo "set shiftwidth=4" >> ~/.vimrc echo "set expandtab" >> ~/.vimrc -# Install IPython to enforce all later command in Python 3 context -$CONDA install -y ipython +# Install dependencies +$CONDA update --all +$CONDA install -y ipython patchelf -# Upgrade pip and install some dependencies pip install --upgrade pip -pip install cmake cython msgpack -pip install setuptools wheel twine -pip install pyglet - -# Install some conda packages -$CONDA install -y numpy scipy matplotlib scikit-image scikit-learn - -# From PyTorch Github README -$CONDA install -y pyyaml mkl mkl-include cffi typing - -# Install patchelf useful for mujoco-py to work, potential problem with Ubuntu 16.04 -# TODO: test on Ubuntu 18.04 -$CONDA install -y patchelf - -# Install some pip packages -pip install pandas seaborn -pip install jupyterlab cloudpickle -pip install mujoco-py gym[all] cma -pip install pytest flake8 sphinx sphinx_rtd_theme - -# PyTorch -$CONDA install pytorch torchvision cuda92 -c pytorch \ No newline at end of file +pip install -r ../requirements.txt diff --git a/setup.py b/setup.py index e62f03cc..cedfd147 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ long_description = f.read() setup(name='lagom', - version=__version__, + version=str(__version__), author='Xingdong Zuo', author_email='zuoxingdong@hotmail.com', description='lagom: A light PyTorch infrastructure to quickly prototype reinforcement learning algorithms.', @@ -36,7 +36,6 @@ 'pyyaml', 'colored'], tests_require=['pytest'], - # Only Python 3+ python_requires='>=3', # List all lagom packages (folder with __init__.py), useful to distribute a release packages=find_packages(), @@ -46,6 +45,4 @@ 'Operating System :: OS Independent', 'Natural Language :: English', 'Topic :: Scientific/Engineering :: Artificial Intelligence'] - - )