Update scripts: cleaner design and all dependencies in requirements.txt

Former-commit-id: c16febf
zuoxingdong · Oct 11, 2018 · 483f3bb · 483f3bb
1 parent aaf0995
commit 483f3bb
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 88 deletions.
diff --git a/readthedocs.yml b/readthedocs.yml
@@ -1,8 +1,6 @@
-# .readthedocs.yml
-
 build:
   image: latest
 
-# Use Python 3.6, specifically to support f-string. 
+# TODO: change to Python 3.7 once ReadtheDocs approves the PR
 python:
   version: 3.6
diff --git a/requirements.txt b/requirements.txt
@@ -1,21 +1,45 @@
-cloudpickle>=0.5.3
-cython>=0.28.4
-pyyaml>=3.12
-setuptools>=39.0.1
-numpy>=1.14.5
+# Update: 2018-10-11
+
+# System-related
+mkl>=2019.0
+mkl-include>=2019.0
+cmake>=3.12.0
+cython>=0.28.5
+cffi>=1.11.5
+typing>=3.6.6
+msgpack>=0.5.6
+setuptools>=40.4.3
+wheel>=0.32.1
+twine>=1.12.1
+
+# Common
+jupyterlab >= 0.35.0
+numpy>=1.15.2
 scipy>=1.1.0
-matplotlib>=2.2.2
-scikit-image>=0.14.0
-scikit-learn>=0.19.2
-imageio>=2.3.0
+pandas>=0.23.4
+cloudpickle>=0.6.1
+pyyaml>=3.13
+colored>=1.3.93
+
+# Testing, PEP8 code quality
+pytest>=3.8.2
+flake8>=3.5.0
+
+# Image, Video
+scikit-image>=0.14.1
+scikit-learn>=0.20.0
+pillow>=5.3.0
+imageio>=2.4.1
 pyglet>=1.3.2
-pandas>=0.23.3
-seaborn>=0.8.1
-jupyterlab >= 0.32.1
-gym>= 0.10.5
+
+# Plotting
+matplotlib>=3.0.0
+seaborn>=0.9.0
+
+# RL
+gym>= 0.10.8
 cma>=2.6.0
-pytest>=3.6.3
-flake8>=3.5.0
-sphinx>=1.7.6
-sphinx_rtd_theme>=0.4.0
-colored>=1.3.93
+
+# ReadtheDocs
+sphinx>=1.8.1
+sphinx_rtd_theme>=0.4.2
diff --git a/scripts/0_install_nvidia_kit.sh b/scripts/0_install_nvidia_kit.sh
@@ -3,24 +3,22 @@
 
 ########################################
 # Install NVIDIA Drivers/CUDA/cuDNN    #
-# Date: 2018-08-10                     #
+# Date: 2018-10-11                     #
 # Author: Xingdong Zuo                 #
 ########################################
 
 # Stop when some intermediate command is failed
 set -e
 
-# Create list of variables
-# Find corresponding Driver runfile from Advanced Driver Search (with Beta version)
-# http://www.nvidia.com/Download/Find.aspx
+# Download GPU Driver file: http://www.nvidia.com/Download/Find.aspx
 export DRIVER_URL=http://us.download.nvidia.com/XFree86/Linux-x86_64/396.51/NVIDIA-Linux-x86_64-396.51.run
 export DRIVER_RUNFILE=NVIDIA-Linux-x86_64-396.51.run
-# Find CUDA runfile from NVIDIA website
-# https://developer.nvidia.com/cuda-downloads
+export DRIVER_VERSION=396
+# Download CUDA runfile: https://developer.nvidia.com/cuda-downloads
 export CUDA_URL=https://developer.nvidia.com/compute/cuda/9.2/Prod2/local_installers/cuda_9.2.148_396.37_linux
 export CUDA_RUNFILE=cuda_9.2.148_396.37_linux
 export CUDA_VERSION=9.2
-# cuDNN file: Download from https://developer.nvidia.com/rdp/cudnn-download
+# Download cuDNN file: https://developer.nvidia.com/rdp/cudnn-download
 export CUDNN_FILE=cudnn-9.2-linux-x64-v7.2.1.38.tgz
 
 # Disable nouveau driver: otherwise distribution provided pre-install will fail
@@ -60,6 +58,13 @@ chmod +x ./$DRIVER_RUNFILE  # make it executable
 sudo ./$DRIVER_RUNFILE --dkms --no-opengl-files  # no OpenGL to support xvfb for fake screen on server
 echo 'NVIDIA driver installation is completed !'
 
+# Append environment variables to .bashrc
+echo "# Appended by Nvidia Driver installer" >> ~/.bashrc
+echo "export PATH=\$PATH:/usr/lib/nvidia-$DRIVER_VERSION/bin/" >> ~/.bashrc
+echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/lib/nvidia-$DRIVER_VERSION/" >> ~/.bashrc
+echo ""  >> ~/.bashrc
+. ~/.bashrc  # refresh bashrc
+
 # Remove driver runfile
 rm ./$DRIVER_RUNFILE
 

diff --git a/scripts/2_install_Anaconda.sh b/scripts/2_install_Anaconda.sh
@@ -2,16 +2,16 @@
 
 #############################
 # Install Anaconda with env #
-# Date: 2018-09-07  	    #
+# Date: 2018-10-11  	    #
 # Author: Xingdong Zuo      #
 #############################
 
 # Stop when some intermediate command is failed
 set -e
 
 # Create list of variables
-export ANACONDA=Anaconda3-5.2.0-Linux-x86_64.sh  # Check new version
-export ENV_NAME=RL  # name of Anaconda environment
+export ANACONDA=Anaconda3-5.3.0-Linux-x86_64.sh  # Check new version
+export ENV_NAME=lagom  # name of Anaconda environment
 export PYTHON_VERSION=3.7  # Python version
 
 # Download and install Anaconda
@@ -21,10 +21,6 @@ chmod +x ./$ANACONDA  # make it executable
 ./$ANACONDA  # New version: don't choose yes to append PATH variable
 rm ./$ANACONDA  # remove the installation file
 
-# Append to load conda command from next run
-# Remove PATH part in bashrc
-echo ". $HOME/anaconda3/etc/profile.d/conda.sh" >> ~/.bashrc
-
 # Alias for conda command
 export CONDA=$HOME/anaconda3/bin/conda
 
@@ -34,11 +30,17 @@ $CONDA update -n base conda
 # Create an Anaconda environment with latest Python
 $CONDA create -n $ENV_NAME python=$PYTHON_VERSION
 
-# Append to bashrc to automatically load virtual environment for each start
-echo "conda activate $ENV_NAME  # automatically activate virtual envs" >> ~/.bashrc  # double quotation works for variable
+# Append environment variables to .bashrc
+echo "# Appended by Anaconda installer" >> ~/.bashrc
+echo ". $HOME/anaconda3/etc/profile.d/conda.sh" >> ~/.bashrc  # Load conda command
+echo "export LIBRARY_PATH=\$LIBRARY_PATH:$HOME/anaconda3/envs/$ENV_NAME/lib" >> ~/.bashrc
+echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:$HOME/anaconda3/envs/$ENV_NAME/lib" >> ~/.bashrc
+echo "export PKG_CONFIG_PATH=\$PKG_CONFIG_PATH:$HOME/anaconda3/envs/$ENV_NAME/lib/pkgconfig/" >> ~/.bashrc
+echo "conda activate $ENV_NAME  # automatically activate virtual envs" >> ~/.bashrc  # auto-load environment
+echo ""  >> ~/.bashrc
+. ~/.bashrc  # refresh bashrc
 
 # Message to restart the shell
-echo "############################"
+echo "#####################################################################"
 echo "Please restart the shell to automatically load conda and environment."
 echo "Then run the second bash file to set up dependencies."
-
diff --git a/scripts/3_setup_envs.sh b/scripts/3_setup_envs.sh
@@ -2,64 +2,24 @@
 
 #############################
 # Install dependencies      #
-# Date: 2018-08-10  	    #
+# Date: 2018-10-11  	    #
 # Author: Xingdong Zuo      #
 #############################
 
 # Stop when some intermediate command is failed
 set -e
 
-# Create list of variables
-export ENV_NAME=RL  # name of Anaconda environment
-export NVIDIA_DRIVER_VER=396  # major version of Nvidia driver
-
 # Alias of conda command
 export CONDA=$HOME/anaconda3/bin/conda
 
-# Update all installed packages
-$CONDA update --all
-
-# Append more useful things to bashrc
-echo "# PATH" >> ~/.bashrc
-echo "export PATH=/usr/lib/nvidia-$NVIDIA_DRIVER_VER/bin/:\$PATH" >> ~/.bashrc
-
-echo "# LIBRARY" >> ~/.bashrc
-echo "export LIBRARY_PATH=$HOME/anaconda3/envs/$ENV_NAME/lib:\$LIBRARY_PATH" >> ~/.bashrc
-echo "export LD_LIBRARY_PATH=/usr/lib/nvidia-$NVIDIA_DRIVER_VER:$HOME/anaconda3/envs/$ENV_NAME/lib:\$LD_LIBRARY_PATH" >> ~/.bashrc
-echo "export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:$HOME/.mujoco/mjpro150/bin" >> ~/.bashrc 
-
-echo "# PKG_CONFIG" >> ~/.bashrc
-echo "export PKG_CONFIG_PATH=\$PKG_CONFIG_PATH:$HOME/anaconda3/envs/$ENV_NAME/lib/pkgconfig/" >> ~/.bashrc
-
 # Set up .vimrc for tab as 4 spaces
 echo "set tabstop=4" >> ~/.vimrc
 echo "set shiftwidth=4" >> ~/.vimrc
 echo "set expandtab" >> ~/.vimrc
 
-# Install IPython to enforce all later command in Python 3 context
-$CONDA install -y ipython
+# Install dependencies
+$CONDA update --all
+$CONDA install -y ipython patchelf
 
-# Upgrade pip and install some dependencies
 pip install --upgrade pip
-pip install cmake cython msgpack
-pip install setuptools wheel twine
-pip install pyglet
-
-# Install some conda packages
-$CONDA install -y numpy scipy matplotlib scikit-image scikit-learn
-
-# From PyTorch Github README
-$CONDA install -y pyyaml mkl mkl-include cffi typing
-
-# Install patchelf useful for mujoco-py to work, potential problem with Ubuntu 16.04
-# TODO: test on Ubuntu 18.04
-$CONDA install -y patchelf
-
-# Install some pip packages
-pip install pandas seaborn
-pip install jupyterlab cloudpickle
-pip install mujoco-py gym[all] cma
-pip install pytest flake8 sphinx sphinx_rtd_theme
-
-# PyTorch
-$CONDA install pytorch torchvision cuda92 -c pytorch
+pip install -r ../requirements.txt
diff --git a/setup.py b/setup.py
@@ -9,7 +9,7 @@
     long_description = f.read()
 
 setup(name='lagom',
-      version=__version__,
+      version=str(__version__),
       author='Xingdong Zuo',
       author_email='zuoxingdong@hotmail.com',
       description='lagom: A light PyTorch infrastructure to quickly prototype reinforcement learning algorithms.',
@@ -36,7 +36,6 @@
                         'pyyaml', 
                         'colored'],
       tests_require=['pytest'],
-      # Only Python 3+
       python_requires='>=3',
       # List all lagom packages (folder with __init__.py), useful to distribute a release
       packages=find_packages(), 
@@ -46,6 +45,4 @@
                    'Operating System :: OS Independent', 
                    'Natural Language :: English', 
                    'Topic :: Scientific/Engineering :: Artificial Intelligence']
-
-
 )