.gitignore

@@ -11,7 +11,7 @@ __pycache__/
 env/
 build/
 develop-eggs/
-dist/
+# dist/
 downloads/
 eggs/
 .eggs/
@@ -117,3 +117,11 @@ images/test/
 test_save.dat
 
 .visualizations
+
+playground/
+**/tmp
+**/TEMP
+
+*.pkl
+
+images
\ No newline at end of file

.gitlab-ci.yml

@@ -10,42 +10,69 @@ image: themattrix/tox
 ## - AWS_SECRET_ACCESS_KEY
 
 stages:
+    - build_wheel
     - tests
     - integration_testing
     - profiling
     - benchmarks
     - deploy_docs
-cache:
-    paths:
-        - .tox
 
 before_script:
     - echo "Setting Up...."
 
-tests:
+.test-setup: &test-setup
     stage: tests
+    services:
+        - redis
     before_script:
         - apt update
-        - apt install -y libgl1-mesa-glx xvfb graphviz xdg-utils libcairo2-dev libjpeg-dev libgif-dev
+        - apt install -y libgl1-mesa-glx xvfb graphviz libgraphviz-dev xdg-utils libcairo2-dev libjpeg-dev libgif-dev python-pyglet
         - wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/Miniconda3-latest-Linux-x86_64.sh
         - bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /tmp/miniconda3
         - export PATH=/tmp/miniconda3/bin:$PATH
         - pip install tox awscli
         - conda update -n root conda -y
-        - conda install -c conda-forge tox-conda
+        - conda install -c conda-forge tox-conda -y
+
+test-py37:
+    <<: *test-setup
+    script:
+        - xvfb-run tox -v --recreate -e py37
+
+test-py38:
+    <<: *test-setup
+    script:
+        - xvfb-run tox -v --recreate -e py38
+
+test-notebooks:
+    <<: *test-setup
+    script:
+        - xvfb-run tox -v --recreate -e notebooks
+
+test-examples:
+    <<: *test-setup
     script:
-        - xvfb-run tox -v --recreate
+        - xvfb-run tox -v --recreate -e examples
+
+test-docs:
+    <<: *test-setup
+    script:
+        - xvfb-run tox -v --recreate -e docs
+
+test-coverage:
+    <<: *test-setup
+    script:
+        - xvfb-run tox -v --recreate -e coverage
 
 build_and_deploy_docs:
-    image: "python:latest"
     stage: deploy_docs
     only:
         - master
     dependencies:
-        - tests
+        - test-docs
     before_script:
         - apt update
-        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev
+        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev graphviz libgraphviz-dev python-pyglet
         - wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/Miniconda3-latest-Linux-x86_64.sh
         - bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /tmp/miniconda3
         - export PATH=/tmp/miniconda3/bin:$PATH
@@ -64,14 +91,18 @@ build_and_deploy_docs:
 
 profiling:
     dependencies:
-        - tests
+        - test-py38
+        - test-py37
+        - test-notebooks
+        - test-examples
+        - test-coverage
     stage: profiling
     only:
         variables:
             - $BENCHMARKS_AND_PROFILING
     before_script:
         - apt update
-        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev
+        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev graphviz libgraphviz-dev python-pyglet
         - wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/Miniconda3-latest-Linux-x86_64.sh
         - bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /tmp/miniconda3
         - export PATH=/tmp/miniconda3/bin:$PATH
@@ -83,14 +114,18 @@ profiling:
 
 benchmarks:
     dependencies:
-        - tests
+        - test-py38
+        - test-py37
+        - test-notebooks
+        - test-examples
+        - test-coverage
     stage: benchmarks
     only:
         variables:
             - $BENCHMARKS_AND_PROFILING
     before_script:
         - apt update
-        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev
+        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev graphviz libgraphviz-dev python-pyglet
         - wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/Miniconda3-latest-Linux-x86_64.sh
         - bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /tmp/miniconda3
         - export PATH=/tmp/miniconda3/bin:$PATH
@@ -102,9 +137,11 @@ benchmarks:
 
 test_conda_setup:
     stage: integration_testing
+    services:
+        - redis
     before_script:
         - apt update
-        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev
+        - apt install -y libgl1-mesa-glx xvfb xdg-utils libcairo2-dev libjpeg-dev libgif-dev graphviz libgraphviz-dev python-pyglet
         - wget -nv https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O /tmp/Miniconda3-latest-Linux-x86_64.sh
         - bash /tmp/Miniconda3-latest-Linux-x86_64.sh -b -p /tmp/miniconda3
         - export PATH=/tmp/miniconda3/bin:$PATH
@@ -112,4 +149,18 @@ test_conda_setup:
     script:
         - xvfb-run bash getting_started/getting_started.sh
 
-
+build_wheel:
+    image: "python:3.7-slim"
+    stage: build_wheel
+    before_script:
+        - apt update
+        - apt install -y make
+        - pip install -r requirements_dev.txt
+    script:
+        - make dist
+        - export WHEEL_NAME="$( find dist -name 'flatland_rl*.whl' )"
+        - mv "${WHEEL_NAME}" "${WHEEL_NAME/-py2.py3-/-py3-}"
+    artifacts:
+        paths:
+            - dist/flatland_rl*.whl
+        expire_in: 2 mos

README.md

@@ -5,18 +5,22 @@
 
 <p style="text-align:center">
 <img alt="repository" src="https://gitlab.aicrowd.com/flatland/flatland/badges/master/pipeline.svg">
-<img alt="discord" src="https://gitlab.aicrowd.com/flatland/flatland/badges/master/coverage.svg">
+<img alt="coverage" src="https://gitlab.aicrowd.com/flatland/flatland/badges/master/coverage.svg">
 </p>
 
 Flatland is a open-source toolkit for developing and comparing Multi Agent Reinforcement Learning algorithms in little (or ridiculously large!) gridworlds.
 
 [The official documentation](http://flatland.aicrowd.com/) contains full details about the environment and problem statement
 
+Flatland is tested with Python 3.6, 3.7 and 3.8 on modern versions of macOS, Linux and Windows. You may encounter problems with graphical rendering if you use WSL. Your [contribution is welcome](https://flatland.aicrowd.com/misc/contributing.html) if you can help with this!  
+
 🏆 Challenges
 ---
 
 This library was developed specifically for the AIcrowd [Flatland challenges](http://flatland.aicrowd.com/research/top-challenge-solutions.html) in which we strongly encourage you to take part in!
 
+- [Flatland 3 Challenge](https://www.aicrowd.com/challenges/flatland-3) - ONGOING!
+- [AMLD 2021 Challenge](https://www.aicrowd.com/challenges/flatland)
 - [NeurIPS 2020 Challenge](https://www.aicrowd.com/challenges/neurips-2020-flatland-challenge/)
 - [2019 Challenge](https://www.aicrowd.com/challenges/flatland-challenge)
 
@@ -28,7 +32,7 @@ This library was developed specifically for the AIcrowd [Flatland challenges](ht
 Install [Anaconda](https://www.anaconda.com/distribution/) and create a new conda environment:
 
 ```console
-$ conda create python=3.6 --name flatland-rl
+$ conda create python=3.7 --name flatland-rl
 $ conda activate flatland-rl
 ```
 
@@ -55,7 +59,7 @@ $ git clone git@gitlab.aicrowd.com:flatland/flatland.git
 Once you have a copy of the source, install it with:
 
 ```console
-$ python setup.py install
+$ pip install -e .
 ```
 
 ### Test installation
@@ -75,7 +79,7 @@ python setup.py test
 👥 Credits
 ---
 
-This library was developed by [SBB](https://www.sbb.ch/en/), [Deutsche Bahn](https://www.deutschebahn.com/), [AIcrowd](https://www.aicrowd.com/) and [numerous contributors](http://flatland.aicrowd.com/misc/credits.html) and AIcrowd research fellows from the AIcrowd community. 
+This library was developed by [SBB](https://www.sbb.ch/en/), [Deutsche Bahn](https://www.deutschebahn.com/), [SNCF](https://www.sncf.com/en), [AIcrowd](https://www.aicrowd.com/) and [numerous contributors](http://flatland.aicrowd.com/misc/credits.html) and AIcrowd research fellows from the AIcrowd community.
 
 ➕ Contributions
 ---
@@ -91,6 +95,7 @@ Please follow the [Contribution Guidelines](https://flatland.aicrowd.com/misc/co
 🔗 Partners
 ---
 
-<a href="https://sbb.ch" target="_blank" style="margin-right:25px"><img src="https://i.imgur.com/OSCXtde.png" alt="SBB" width="200"/></a> 
-<a href="https://www.deutschebahn.com/" target="_blank" style="margin-right:25px"><img src="https://i.imgur.com/pjTki15.png" alt="DB"  width="200"/></a>
-<a href="https://www.aicrowd.com" target="_blank"><img src="https://avatars1.githubusercontent.com/u/44522764?s=200&v=4" alt="AICROWD"  width="200"/></a>
+<a href="https://sbb.ch" target="_blank" style="margin-right:30px"><img src="https://annpr2020.ch/wp-content/uploads/2020/06/SBB.png" alt="SBB" width="140"/></a> 
+<a href="https://www.deutschebahn.com/" target="_blank" style="margin-right:30px"><img src="https://i.imgur.com/pjTki15.png" alt="DB"  width="140"/></a>
+<a href="https://www.sncf.com/en" target="_blank" style="margin-right:30px"><img src="https://iconape.com/wp-content/png_logo_vector/logo-sncf.png" alt="SNCF"  width="140"/></a>
+<a href="https://www.aicrowd.com" target="_blank"><img src="https://i.imgur.com/kBZQGI9.png" alt="AIcrowd"  width="140"/></a>

benchmarks/benchmark_all_examples.py

@@ -34,4 +34,6 @@ for entry in [entry for entry in importlib_resources.contents('examples') if
                 # https://docs.python.org/3/library/test.html
                 # TODO remove input() from examples?
                 with swap_attr(sys, "stdin", StringIO("q")):
-                    runpy.run_path(file_in, run_name="__main__")
+                    runpy.run_path(file_in, run_name="__main__", init_globals={
+                    'argv': ['--sleep-for-animation=False', '--do_rendering=False']
+                })

benchmarks/profile_all_examples.py

@@ -19,7 +19,9 @@ def profile(resource, entry):
         with swap_attr(sys, "stdin", StringIO("q")):
             global my_func
 
-            def my_func(): runpy.run_path(file_in, run_name="__main__")
+            def my_func(): runpy.run_path(file_in, run_name="__main__", init_globals={
+                    'argv': ['--sleep-for-animation=False', '--do_rendering=False']
+                })
 
             cProfile.run('my_func()', sort='time')
 

benchmarks/run_all_examples.py

@@ -8,8 +8,8 @@ from importlib_resources import path
 
 from benchmarks.benchmark_utils import swap_attr
 
-print("GRRRRRRRR run_all_examples.py")
-
+print("run_all_examples.py")
+error_log_examples = {}
 for entry in [entry for entry in importlib_resources.contents('examples') if
               not pkg_resources.resource_isdir('examples', entry)
               and entry.endswith(".py")
@@ -17,6 +17,7 @@ for entry in [entry for entry in importlib_resources.contents('examples') if
               and 'demo.py' not in entry
               and 'DELETE' not in entry
               ]:
+
     with path('examples', entry) as file_in:
         print("")
         print("")
@@ -25,12 +26,25 @@ for entry in [entry for entry in importlib_resources.contents('examples') if
         print("*****************************************************************")
         print("Running {}".format(entry))
         print("*****************************************************************")
+
         with swap_attr(sys, "stdin", StringIO("q")):
             try:
                 runpy.run_path(file_in, run_name="__main__", init_globals={
-                    'argv': ['--sleep-for-animation=False']
+                    'argv': ['--sleep-for-animation=False', '--do_rendering=False']
                 })
             except Exception as e:
                 print(e)
+                error_log_examples.update({file_in: e})
             print("runpy done.")
         print("Done with {}".format(entry))
+
+if len(error_log_examples.keys()) > 0:
+    print("*****************************************************************")
+    print("Error log:")
+    print("*****************************************************************")
+    print(error_log_examples)
+    print("*****************************************************************")
+else:
+    print("*****************************************************************")
+    print("All examples executed - no error.")
+    print("*****************************************************************")

docs/08_authors.rst

 Authors
 =======
-.. toctree::
-   :maxdepth: 2
 
 .. include:: ../AUTHORS.rst

docs/10_interface.rst

+Multi-Agent Interface
+=======
+
+.. include:: interface/pettingzoo.rst
+.. include:: interface/wrappers.rst
+
+Multi-Agent Pettingzoo Usage
+=======
+
+We can use the PettingZoo interface by proving the rail env to the petting zoo wrapper as shown below in the example.
+
+.. literalinclude:: ../tests/test_pettingzoo_interface.py
+   :language: python
+   :start-after: __sphinx_doc_begin__
+   :end-before: __sphinx_doc_end__
+
+
+Multi-Agent Interface Stable Baseline 3 Training
+=======
+
+.. literalinclude:: ../flatland/contrib/training/flatland_pettingzoo_stable_baselines.py
+   :language: python
+   :start-after: __sphinx_doc_begin__
+   :end-before: __sphinx_doc_end__
+
+
+Multi-Agent Interface Rllib Training
+=======
+
+.. literalinclude:: ../flatland/contrib/training/flatland_pettingzoo_rllib.py
+   :language: python
+   :start-after: __sphinx_doc_begin__
+   :end-before: __sphinx_doc_end__
\ No newline at end of file

docs/10_interface_toc.rst

+Multi-Agent Interfaces
+==============
+
+
+.. toctree::
+   :maxdepth: 2
+
+   10_interface

docs/index.rst

@@ -15,6 +15,7 @@ Welcome to flatland's documentation!
    07_changes
    08_authors
    09_faq_toc
+   10_interface
 
 Indices and tables
 ==================

docs/interface/pettingzoo.md

+# PettingZoo
+
+> PettingZoo (https://www.pettingzoo.ml/) is a collection of multi-agent environments for reinforcement learning. We build a pettingzoo interface for flatland.
+
+## Background
+
+PettingZoo is a popular multi-agent environment library (https://arxiv.org/abs/2009.14471) that aims to be the gym standard for Multi-Agent Reinforcement Learning. We list the below advantages that make it suitable for use with flatland
+
+- Works with both rllib (https://docs.ray.io/en/latest/rllib.html) and stable baselines 3 (https://stable-baselines3.readthedocs.io/) using wrappers from Super Suit.
+- Clean API (https://www.pettingzoo.ml/api) with additional facilities/api for parallel, saving observation, recording using gym monitor, processing, normalising observations
+- Scikit-learn inspired api
+  e.g.
+
+```python
+act = model.predict(obs, deterministic=True)[0] 
+```
+
+- Parallel learning using literally 2 lines of code to use with stable baselines 3
+
+```python
+env = ss.pettingzoo_env_to_vec_env_v0(env)
+env = ss.concat_vec_envs_v0(env, 8, num_cpus=4, base_class=’stable_baselines3’)
+```
+
+- Tested and supports various multi-agent environments with many agents comparable to flatland. e.g. https://www.pettingzoo.ml/magent
+- Clean interface means we can custom add an experimenting tool like wandb and have full flexibility to save information we want

docs/interface/pettingzoo.rst

+
+PettingZoo
+==========
+
+..
+
+   PettingZoo (https://www.pettingzoo.ml/) is a collection of multi-agent environments for reinforcement learning. We build a pettingzoo interface for flatland.
+
+
+Background
+----------
+
+PettingZoo is a popular multi-agent environment library (https://arxiv.org/abs/2009.14471) that aims to be the gym standard for Multi-Agent Reinforcement Learning. We list the below advantages that make it suitable for use with flatland
+
+
+* Works with both rllib (https://docs.ray.io/en/latest/rllib.html) and stable baselines 3 (https://stable-baselines3.readthedocs.io/) using wrappers from Super Suit.
+* Clean API (https://www.pettingzoo.ml/api) with additional facilities/api for parallel, saving observation, recording using gym monitor, processing, normalising observations
+* Scikit-learn inspired api
+  e.g.
+
+.. code-block:: python
+
+   act = model.predict(obs, deterministic=True)[0]
+
+
+* Parallel learning using literally 2 lines of code to use with stable baselines 3
+
+.. code-block:: python
+
+   env = ss.pettingzoo_env_to_vec_env_v0(env)
+   env = ss.concat_vec_envs_v0(env, 8, num_cpus=4, base_class=’stable_baselines3’)
+
+
+* Tested and supports various multi-agent environments with many agents comparable to flatland. e.g. https://www.pettingzoo.ml/magent
+* Clean interface means we can custom add an experimenting tool like wandb and have full flexibility to save information we want

docs/interface/wrappers.md

+# Environment Wrappers
+
+> We provide various environment wrappers to work with both the rail env and the petting zoo interface.
+
+## Background
+
+These wrappers changes certain environment behavior which can help to get better reinforcement learning training.
+
+## Supported Inbuilt Wrappers
+
+We provide 2 sample wrappers for ShortestPathAction wrapper and SkipNoChoice wrapper. The wrappers requires many env properties that are only created on environment reset. Hence before using the wrapper, we must reset the rail env. To use the wrappers, simply pass the resetted rail env. Code samples are shown below for each wrapper.
+
+### ShortestPathAction Wrapper
+
+To use the ShortestPathAction Wrapper, simply wrap the rail env as follows
+
+```python
+rail_env.reset(random_seed=1)
+rail_env = ShortestPathActionWrapper(rail_env)
+```
+
+The shortest path action wrapper maps the existing action space into 3 actions - Shortest Path (`0`), Next Shortest Path (`1`) and Stop (`2`).  Hence, we must ensure that the predicted action should always be one of these (0, 1 and 2) actions. To route all agents in the shortest path, pass `0` as the action.
+
+### SkipNoChoice Wrapper
+
+To use the SkipNoChoiceWrapper, simply wrap the rail env as follows
+
+```python
+rail_env.reset(random_seed=1)
+rail_env = SkipNoChoiceCellsWrapper(rail_env, accumulate_skipped_rewards=False, discounting=0.0)
+```

docs/interface/wrappers.rst

+
+Environment Wrappers
+====================
+
+..
+
+   We provide various environment wrappers to work with both the rail env and the petting zoo interface.
+
+
+Background
+----------
+
+These wrappers changes certain environment behavior which can help to get better reinforcement learning training.
+
+Supported Inbuilt Wrappers
+--------------------------
+
+We provide 2 sample wrappers for ShortestPathAction wrapper and SkipNoChoice wrapper. The wrappers requires many env properties that are only created on environment reset. Hence before using the wrapper, we must reset the rail env. To use the wrappers, simply pass the resetted rail env. Code samples are shown below for each wrapper.
+
+ShortestPathAction Wrapper
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To use the ShortestPathAction Wrapper, simply wrap the rail env as follows
+
+.. code-block:: python
+
+   rail_env.reset(random_seed=1)
+   rail_env = ShortestPathActionWrapper(rail_env)
+
+The shortest path action wrapper maps the existing action space into 3 actions - Shortest Path (\ ``0``\ ), Next Shortest Path (\ ``1``\ ) and Stop (\ ``2``\ ).  Hence, we must ensure that the predicted action should always be one of these (0, 1 and 2) actions. To route all agents in the shortest path, pass ``0`` as the action.
+
+SkipNoChoice Wrapper
+^^^^^^^^^^^^^^^^^^^^
+
+To use the SkipNoChoiceWrapper, simply wrap the rail env as follows
+
+.. code-block:: python
+
+   rail_env.reset(random_seed=1)
+   rail_env = SkipNoChoiceCellsWrapper(rail_env, accumulate_skipped_rewards=False, discounting=0.0)

env_data/tests/service_test/metadata.csv

+test_id,env_id,n_agents,x_dim,y_dim,n_cities,max_rail_pairs_in_city,n_envs_run,seed,grid_mode,max_rails_between_cities,malfunction_duration_min,malfunction_duration_max,malfunction_interval,speed_ratios
+Test_0,Level_0,7,30,30,2,2,10,335971,False,2,20,50,540,"{1.0: 0.25, 0.5: 0.25, 0.33: 0.25, 0.25: 0.25}"
+Test_0,Level_1,7,30,30,2,2,10,335972,False,2,20,50,540,"{1.0: 0.25, 0.5: 0.25, 0.33: 0.25, 0.25: 0.25}"