Merge branch '97-use-tox-conda' into 'master'

Resolve "Use tox-conda to setup environment."

See merge request flatland/flatland!102

.gitignore

@@ -71,8 +71,6 @@ target/
 # Jupyter Notebook
 .ipynb_checkpoints
 
-# Jupyter Notebooks converted to python
-notebooks/*.py
 
 # PyCharm
 .idea/

.gitlab-ci.yml

@@ -11,6 +11,7 @@ image: themattrix/tox
 
 stages:
     - tests
+    - integration_testing
     - benchmarks_and_profiling
     - deploy_docs
 cache:
@@ -22,13 +23,16 @@ before_script:
 
 tests:
     stage: tests
-    script:
+    before_script:
         - apt update
         - apt install -y libgl1-mesa-glx xvfb graphviz xdg-utils libcairo2-dev libjpeg-dev libgif-dev
-        - wget -nv https://repo.anaconda.com/archive/Anaconda3-5.3.1-Linux-x86_64.sh -O /tmp/Anaconda3-5.3.1-Linux-x86_64.sh
-        - bash /tmp/Anaconda3-5.3.1-Linux-x86_64.sh -b -p /tmp/anaconda3
-        - export PATH=/tmp/anaconda3/bin:$PATH; bash getting_started/getting_started.sh
-        - pip install tox
+        - wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/Miniconda3-latest-Linux-x86_64.sh
+        - bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /tmp/miniconda3
+        - export PATH=/tmp/miniconda3/bin:$PATH
+        - pip install tox awscli
+        - conda update -n root conda -y
+        - conda install -c conda-forge tox-conda
+    script:
         - xvfb-run tox -v --recreate
 
 build_and_deploy_docs:
@@ -40,8 +44,13 @@ build_and_deploy_docs:
         - tests
     before_script:
         - apt update
-        - apt install -y graphviz libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev
+        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev
+        - wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/Miniconda3-latest-Linux-x86_64.sh
+        - bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /tmp/miniconda3
+        - export PATH=/tmp/miniconda3/bin:$PATH
         - pip install tox awscli
+        - conda update -n root conda -y
+        - conda install -c conda-forge tox-conda
     script:
         - echo "Bucket=${BUCKET_NAME}"
         - echo "AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION}"
@@ -59,10 +68,28 @@ benchmarks_and_profiling:
     only:
         variables:
             - $BENCHMARKS_AND_PROFILING
-    script:
+    before_script:
         - apt update
-        - apt install -y libgl1-mesa-glx xvfb graphviz xdg-utils libcairo2-dev libjpeg-dev libgif-dev
-        - pip install tox
+        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev
+        - wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/Miniconda3-latest-Linux-x86_64.sh
+        - bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /tmp/miniconda3
+        - export PATH=/tmp/miniconda3/bin:$PATH
+        - pip install tox awscli
+        - conda update -n root conda -y
+        - conda install -c conda-forge tox-conda
+    script:
         - xvfb-run tox -e benchmarks,profiling -v --recreate
 
+test_conda_setup:
+    stage: integration_testing
+    before_script:
+        - apt update
+        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev
+        - wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/Miniconda3-latest-Linux-x86_64.sh
+        - bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /tmp/miniconda3
+        - export PATH=/tmp/miniconda3/bin:$PATH
+        - conda update -n root conda -y
+    script:
+        - xvfb-run bash getting_started/getting_started.sh
+
 

CONTRIBUTING.rst

@@ -64,20 +64,19 @@ Ready to contribute? Here's how to set up `flatland` for local development.
 
     $ git clone git@gitlab.aicrowd.com:flatland/flatland.git
 
-3. Install the software dependencies via Anaconda. (This assumes you have Anaconda installed by following the instructions `here <https://www.anaconda.com/distribution>`_)
+3. Install the software dependencies via Anaconda-3 or Miniconda-3. (This assumes you have Anaconda installed by following the instructions `here <https://www.anaconda.com/distribution>`_)
 
-    $ conda create python=3.6 --name flatland-rl
-    $ conda activate flatland-rl
-    $ conda install -c conda-forge cairosvg pycairo
-    $ conda install -c anaconda tk
-    $ cd flatland
-    $ pip install -e .
+    $ conda install -c conda-forge tox-conda
+    $ conda install tox
+    $ tox -v --recreate
+
+    This will create a virtual env you can then use.
 
     These steps are performed if you run
 
     $ getting_started/getting_started.bat/.sh
 
-    from Anaconda prompt. Only prerequisites: git and Anaconda.
+    from Anaconda prompt.
 
 
 4. Create a branch for local development::

getting_started/getting_started.bat

@@ -2,50 +2,12 @@
 
 set FLATLAND_BASEDIR=%~dp0\..
 
-@echo off
-echo "************ TESTING PREREQUISITES PYTHON3 + GIT + GIT *************************"
-@echo on
-
-git --version || goto :error
-python --version || goto :error
-rem deactivate in case we're in virtualenv
-call conda deactivate || call deactivate
-
-
-@echo off
-echo "************ SETUP VIRTUAL ENVIRONMENT FLATLAND *************************"
-@echo on
-(conda info --envs | findstr flatland-rl) || conda create python=3.6 -y --name flatland-rl || goto :error
-(call conda activate flatland-rl || call activate flatland-rl) || goto :error
-
-
-@echo off
-echo "************ INSTALL FLATLAND AND DEPENDENCIES IN THE VIRTUALENV  *************************"
-@echo on
-rem TODO we should get rid of having to install these packages outside of setup.py with conda!
-call conda install -y -c conda-forge cairosvg pycairo || goto :error
-call conda install -y -c anaconda tk || goto :error
-call python -m pip install --upgrade pip || goto :error
-python setup.py install || goto :error
-
-# ensure jupyter is installed in the virtualenv
-python -m pip install --upgrade -r %FLATLAND_BASEDIR%/requirements_dev.txt -r %FLATLAND_BASEDIR%/requirements_continuous_integration.txt || goto :error
-
-
-@echo off
-echo "************ INSTALL JUPYTER EXTENSION *************************"
-@echo on
-jupyter nbextension install --py --sys-prefix widgetsnbextension || goto :error
-jupyter nbextension enable --py --sys-prefix widgetsnbextension || goto :error
-jupyter nbextension install --py --sys-prefix jpy_canvas || goto :error
-jupyter nbextension enable --py --sys-prefix jpy_canvas || goto :error
-
-
-@echo off
-echo "************ RUN JUPYTER NOTEBOOKS *************************"
-@echo on
-jupyter notebook || goto :error
+cd %FLATLAND_BASEDIR%
 
+call conda install -y -c conda-forge tox-conda || goto :error
+call conda install -y tox || goto :error
+call tox -v  --recreate || goto :error
+call tox -v -e start_jupyter --recreate || goto :error
 
 goto :EOF
 

getting_started/getting_started.sh

@@ -3,54 +3,10 @@ set -e # stop on error
 set -x # echo commands
 
 
-# https://stackoverflow.com/questions/4774054/reliable-way-for-a-bash-script-to-get-the-full-path-to-itself
-FLATLAND_BASEDIR=$(dirname "$0")/..
-FLATLAND_BASEDIR=$(realpath "$FLATLAND_BASEDIR")
-echo "BASEDIR=${FLATLAND_BASEDIR}"
+FLATLAND_BASEDIR=$(dirname "$BASH_SOURCE")/..
+cd ${FLATLAND_BASEDIR}
 
-set +x
-echo "************ TESTING PREREQUISITES PYTHON3 + GIT *************************"
-set -x
-
-git --version
-python --version
-conda --version
-echo $PATH
-
-
-set +x
-echo "************ SETUP VIRTUAL ENVIRONMENT FLATLAND *************************"
-set -x
-source deactivate
-(conda info --envs | fgrep flatland-rl) || conda create python=3.6 -y --name flatland-rl
-source activate flatland-rl
-
-
-set +x
-echo "************ INSTALL FLATLAND IN THE VIRTUALENV  *************************"
-set -x
-
-# TODO we should get rid of having to install these packages outside of setup.py with conda!
-conda install -y -c conda-forge cairosvg pycairo
-conda install -y  -c anaconda tk
-python -m pip install --upgrade pip
-
-python ${FLATLAND_BASEDIR}/setup.py install
-
-# ensure jupyter is installed in the virtualenv
-python -m pip install --upgrade -r ${FLATLAND_BASEDIR}/requirements_dev.txt -r requirements_continuous_integration.txt
-
-
-set +x
-echo "************ INSTALL JUPYTER EXTENSION *************************"
-set -x
-jupyter nbextension install --py --sys-prefix widgetsnbextension
-jupyter nbextension enable --py --sys-prefix widgetsnbextension
-jupyter nbextension install --py --sys-prefix jpy_canvas
-jupyter nbextension enable --py --sys-prefix jpy_canvas
-
-
-set +x
-echo "************ RUN JUPYTER NOTEBOOKS *************************"
-set -x
-jupyter notebook &
+conda install -y -c conda-forge tox-conda
+conda install -y tox
+tox -v
+tox -v -e start_jupyter &

notebooks/run_all_notebooks.py

+import shlex
+import sys
+from subprocess import Popen, PIPE
+
+import importlib_resources
+import pkg_resources
+from importlib_resources import path
+from ipython_genutils.py3compat import string_types, bytes_to_str
+
+
+# taken from https://github.com/jupyter/nbconvert/blob/master/nbconvert/tests/base.py
+def run_python(parameters, ignore_return_code=False, stdin=None):
+    """
+    Run python as a shell command, listening for both Errors and
+    non-zero return codes. Returns the tuple (stdout, stderr) of
+    output produced during the nbconvert run.
+    Parameters
+    ----------
+    parameters : str, list(str)
+        List of parameters to pass to IPython.
+    ignore_return_code : optional bool (default False)
+        Throw an OSError if the return code
+    """
+    cmd = [sys.executable]
+    if sys.platform == 'win32':
+        if isinstance(parameters, string_types):
+            cmd = ' '.join(cmd) + ' ' + parameters
+        else:
+            cmd = ' '.join(cmd + parameters)
+    else:
+        if isinstance(parameters, string_types):
+            parameters = shlex.split(parameters)
+        cmd += parameters
+    p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
+    stdout, stderr = p.communicate(input=stdin)
+    if not (p.returncode == 0 or ignore_return_code):
+        raise OSError(bytes_to_str(stderr))
+    return stdout.decode('utf8', 'replace'), stderr.decode('utf8', 'replace')
+
+
+for entry in [entry for entry in importlib_resources.contents('notebooks') if
+              not pkg_resources.resource_isdir('notebooks', entry)
+              and entry.endswith(".ipynb")
+              ]:
+    print("*****************************************************************")
+    print("Converting and running {}".format(entry))
+    print("*****************************************************************")
+
+    with path('notebooks', entry) as file_in:
+        out, err = run_python(" -m jupyter nbconvert --execute --to notebook --inplace " + str(file_in))
+        sys.stderr.write(err)
+        sys.stderr.flush()
+        sys.stdout.write(out)
+        sys.stdout.flush()

requirements_dev.txt

@@ -15,4 +15,7 @@ svgutils>=0.3.1
 screeninfo>=0.3.1
 pyarrow>=0.13.0
 importlib-metadata>=0.17
-importlib_resources>=1.0.2
+importlib-resources>=1.0.1
+six>=1.12.0
+attrs
+ushlex

setup.py

@@ -14,17 +14,25 @@ with open('README.rst') as readme_file:
 
 def get_all_svg_files(directory='./svg/'):
     ret = []
-    for f in os.listdir(directory):
-        if os.path.isfile(os.path.join(directory, f)):
-            ret.append(directory + f)
+    for dirpath, subdirs, files in os.walk(directory):
+        for f in files:
+            ret.append(os.path.join(dirpath,f))
     return ret
 
 
 def get_all_images_files(directory='./images/'):
     ret = []
-    for f in os.listdir(directory):
-        if os.path.isfile(os.path.join(directory, f)):
-            ret.append(directory + f)
+    for dirpath, subdirs, files in os.walk(directory):
+        for f in files:
+            ret.append(os.path.join(dirpath,f))
+    return ret
+
+
+def get_all_notebook_files(directory='./notebooks/'):
+    ret = []
+    for dirpath, subdirs, files in os.walk(directory):
+        for f in files:
+            ret.append(os.path.join(dirpath,f))
     return ret
 
 
@@ -63,7 +71,9 @@ setup(
     keywords='flatland',
     name='flatland-rl',
     packages=find_packages('.'),
-    data_files=[('svg', get_all_svg_files()), ('images', get_all_images_files())],
+    data_files=[('svg', get_all_svg_files()),
+                ('images', get_all_images_files()),
+                ('notebooks', get_all_notebook_files())],
     setup_requires=setup_requirements,
     test_suite='tests',
     tests_require=test_requirements,

tests/test_flatland_envs_observations.py

@@ -10,7 +10,7 @@ from flatland.envs.observations import GlobalObsForRailEnv, TreeObsForRailEnv
 from flatland.envs.predictions import ShortestPathPredictorForRailEnv
 from flatland.envs.rail_env import RailEnv, RailEnvActions
 from flatland.utils.rendertools import RenderTool
-from tests.simple_rail import make_simple_rail
+from flatland.utils.simple_rail import make_simple_rail
 
 """Tests for `flatland` package."""
 

tests/test_flatland_envs_predictions.py

@@ -10,7 +10,7 @@ from flatland.envs.observations import TreeObsForRailEnv
 from flatland.envs.predictions import DummyPredictorForRailEnv, ShortestPathPredictorForRailEnv
 from flatland.envs.rail_env import RailEnv
 from flatland.utils.rendertools import RenderTool
-from tests.simple_rail import make_simple_rail
+from flatland.utils.simple_rail import make_simple_rail
 
 """Test predictions for `flatland` package."""
 

tox.ini

 [tox]
 envlist = py36, py37, examples, notebooks, flake8, docs, coverage
 
+
 [travis]
 python =
     3.7: py37
@@ -20,34 +21,58 @@ commands =
     flake8 flatland tests examples benchmarks
 
 [testenv:docs]
-basepython = python
+; TODO https://gitlab.aicrowd.com/flatland/flatland/issues/33 docs requires make installed, therefore exclude Windows
+platform = linux|linux2|darwin
+; use python3.6 because of incompatibility under Windows of the pycairo installed through conda for py37
+basepython = python3.6
 whitelist_externals = make
 passenv =
     DISPLAY
     HTTP_PROXY
     HTTPS_PROXY
+conda_deps =
+    cairosvg
+    pycairo
+    tk
+    graphviz
+conda_channels :
+    conda-forge
+    anaconda
 deps =
     -r{toxinidir}/requirements_dev.txt
     -r{toxinidir}/requirements_continuous_integration.txt
+changedir = {toxinidir}
 commands =
     make docs
 
 [testenv:coverage]
-basepython = python
+; TODO https://gitlab.aicrowd.com/flatland/flatland/issues/33 coverage requires make, therefore exclude Windows
+platform = linux|linux2|darwin
+; use python3.6 because of incompatibility under Windows of the pycairo installed through conda for py37
+basepython = python3.6
 whitelist_externals = make
 passenv =
     DISPLAY
 ; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
     HTTP_PROXY
     HTTPS_PROXY
+conda_deps =
+    cairosvg
+    pycairo
+    tk
+conda_channels :
+    conda-forge
+    anaconda
 deps =
     -r{toxinidir}/requirements_dev.txt
     -r{toxinidir}/requirements_continuous_integration.txt
+changedir = {toxinidir}
 commands =
     make coverage
 
 [testenv:benchmarks]
-basepython = python
+; use python3.6 because of incompatibility under Windows of the pycairo installed through conda for py37
+basepython = python3.6
 setenv =
     PYTHONPATH = {toxinidir}
 passenv =
@@ -61,10 +86,12 @@ deps =
     -r{toxinidir}/requirements_dev.txt
     -r{toxinidir}/requirements_continuous_integration.txt
 commands =
+    python --version
     python benchmarks/benchmark_all_examples.py
 
 [testenv:profiling]
-basepython = python
+; use python3.6 because of incompatibility under Windows of the pycairo installed through conda for py37
+basepython = python3.6
 setenv =
     PYTHONPATH = {toxinidir}
 passenv =
@@ -73,7 +100,13 @@ passenv =
 ; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
     HTTP_PROXY
     HTTPS_PROXY
-whitelist_externals = sh
+conda_deps =
+    cairosvg
+    pycairo
+    tk
+conda_channels :
+    conda-forge
+    anaconda
 deps =
     -r{toxinidir}/requirements_dev.txt
     -r{toxinidir}/requirements_continuous_integration.txt
@@ -81,7 +114,9 @@ commands =
     python benchmarks/profile_all_examples.py
 
 [testenv:examples]
-basepython = python
+; TODO should examples be run with py36 and py37??
+; use python3.6 because of incompatibility under Windows of the pycairo installed through conda for py37
+basepython = python3.6
 setenv =
     PYTHONPATH = {toxinidir}
 passenv =
@@ -90,16 +125,24 @@ passenv =
 ; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
     HTTP_PROXY
     HTTPS_PROXY
-whitelist_externals = sh
+conda_deps =
+    cairosvg
+    pycairo
+    tk
+conda_channels :
+    conda-forge
+    anaconda
 deps =
     -r{toxinidir}/requirements_dev.txt
+; run tests from subfolder to ensure that resources are accessed via resources and not via relative paths
+changedir = {envtmpdir}/c236d3c240d61a0969d4cb59e2180ce5
 commands =
-; run examples from subfolder to ensure that resources are accessed via resources and not via relative paths
-    sh -c 'mkdir -p {envtmpdir}/c236d3c240d61a0969d4cb59e2180ce5'
-    sh -c 'cd {envtmpdir}/c236d3c240d61a0969d4cb59e2180ce5 &&  python {toxinidir}/benchmarks/run_all_examples.py'
+    python {toxinidir}/benchmarks/run_all_examples.py
 
 [testenv:notebooks]
-basepython = python
+; TODO should examples be run with py36 and py37??
+; use python3.6 because of incompatibility under Windows of the pycairo installed through conda for py37
+basepython = python3.6
 setenv =
     PYTHONPATH = {toxinidir}
 passenv =
@@ -112,20 +155,53 @@ whitelist_externals = sh
 deps =
     -r{toxinidir}/requirements_dev.txt
     -r{toxinidir}/requirements_continuous_integration.txt
-commands =
+conda_deps =
+    cairosvg
+    pycairo
+    tk
+conda_channels :
+    conda-forge
+    anaconda
 ; run tests from subfolder to ensure that resources are accessed via resources and not via relative paths
-    sh -c 'mkdir -p {envtmpdir}/6f59bc68108c3895b1828abdd04b9a06'
-    sh -c 'jupyter nbextension install --py --sys-prefix widgetsnbextension'
-    sh -c 'jupyter nbextension enable --py --sys-prefix widgetsnbextension'
-    sh -c 'jupyter nbextension install --py --sys-prefix jpy_canvas'
-    sh -c 'jupyter nbextension enable --py --sys-prefix jpy_canvas'
-; https://stackoverflow.com/questions/35545402/how-to-run-an-ipynb-jupyter-notebook-from-terminal/35545463
-    sh -c 'ls notebooks/*.ipynb  | xargs -n 1 jupyter nbconvert --to python'
-    sh -c 'cd {envtmpdir}/6f59bc68108c3895b1828abdd04b9a06 && ls {toxinidir}/notebooks/*.py  | xargs -I{} -n 1 sh -c "echo -e \"\n====== Running {} ========\n\"; ipython {}"'
+changedir = {envtmpdir}/6f59bc68108c3895b1828abdd04b9a06
+commands =
+    python -m jupyter nbextension install --py --sys-prefix widgetsnbextension
+    python -m jupyter nbextension enable --py --sys-prefix widgetsnbextension
+    python -m jupyter nbextension install --py --sys-prefix jpy_canvas
+    python -m jupyter nbextension enable --py --sys-prefix jpy_canvas
+    python {toxinidir}/notebooks/run_all_notebooks.py
 
-[testenv]
+[testenv:start_jupyter]
+; use python3.6 because of incompatibility under Windows of the pycairo installed through conda for py37
+basepython = python3.6
+setenv =
+    PYTHONPATH = {toxinidir}
+passenv =
+    DISPLAY
+    XAUTHORITY
+; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
+    HTTP_PROXY
+    HTTPS_PROXY
 whitelist_externals = sh
-                      pip
+deps =
+    -r{toxinidir}/requirements_dev.txt
+    -r{toxinidir}/requirements_continuous_integration.txt
+conda_deps =
+    cairosvg
+    pycairo
+    tk
+conda_channels :
+    conda-forge
+    anaconda
+changedir = {toxinidir}
+commands =
+    python -m jupyter nbextension install --py --sys-prefix widgetsnbextension
+    python -m jupyter nbextension enable --py --sys-prefix widgetsnbextension
+    python -m jupyter nbextension install --py --sys-prefix jpy_canvas
+    python -m jupyter nbextension enable --py --sys-prefix jpy_canvas
+    python -m jupyter notebook
+
+[testenv]
 setenv =
     PYTHONPATH = {toxinidir}
 passenv =
@@ -134,9 +210,43 @@ passenv =
 ; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
     HTTP_PROXY
     HTTPS_PROXY
+conda_deps =
+    cairosvg
+    pycairo
+    tk
+conda_channels :
+    conda-forge
+    anaconda
 deps =
     -r{toxinidir}/requirements_dev.txt
+; run tests from subfolder to ensure that resources are accessed via resources and not via relative paths
+changedir = {envtmpdir}/fefed3ba12bf1ed81dbcc20fb52706ea
 commands =
+    python --version
+    python -m pytest --basetemp={envtmpdir} {toxinidir}
+
+[testenv:py37]
+; exclude py37 from Windows because of incompatibility the pycairo installed through conda for py37
+platform = linux|linux2|darwin
+setenv =
+    PYTHONPATH = {toxinidir}
+passenv =
+    DISPLAY
+    XAUTHORITY
+; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
+    HTTP_PROXY
+    HTTPS_PROXY
+conda_deps =
+    cairosvg
+    pycairo
+    tk
+conda_channels :
+    conda-forge
+    anaconda
+deps =
+    -r{toxinidir}/requirements_dev.txt
 ; run tests from subfolder to ensure that resources are accessed via resources and not via relative paths
-    sh -c 'mkdir -p {envtmpdir}/fefed3ba12bf1ed81dbcc20fb52706ea'
-    sh -c 'cd {envtmpdir}/fefed3ba12bf1ed81dbcc20fb52706ea && py.test --basetemp={envtmpdir} {toxinidir}'
+changedir = {envtmpdir}/fefed3ba12bf1ed81dbcc20fb52706ea
+commands =
+    python --version
+    python -m pytest --basetemp={envtmpdir} {toxinidir}