diff --git a/docs/conf.py b/docs/conf.py index 8223e1c7238176cccaaa116c0776bf80438750c7..2ec79aa8f3288861d82ce2b2f20d4cd1329d323c 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -33,7 +33,7 @@ sys.path.insert(0, os.path.abspath('..')) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode'] +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx'] # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/docs/flatland.evaluators.rst b/docs/flatland.evaluators.rst new file mode 100644 index 0000000000000000000000000000000000000000..022e8586b7933af4f7eac78a01168c27639c65c2 --- /dev/null +++ b/docs/flatland.evaluators.rst @@ -0,0 +1,46 @@ +flatland.evaluators package +=========================== + +Submodules +---------- + +flatland.evaluators.aicrowd\_helpers module +------------------------------------------- + +.. automodule:: flatland.evaluators.aicrowd_helpers + :members: + :undoc-members: + :show-inheritance: + +flatland.evaluators.client module +--------------------------------- + +.. automodule:: flatland.evaluators.client + :members: + :undoc-members: + :show-inheritance: + +flatland.evaluators.messages module +----------------------------------- + +.. automodule:: flatland.evaluators.messages + :members: + :undoc-members: + :show-inheritance: + +flatland.evaluators.service module +---------------------------------- + +.. automodule:: flatland.evaluators.service + :members: + :undoc-members: + :show-inheritance: + + +Module contents +--------------- + +.. automodule:: flatland.evaluators + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/flatland.rst b/docs/flatland.rst index fce09a2c0f1f2b442b39d6e51f160fce7981b757..88e8ec93fd4f6c89c1f6e20c55defbddaa9b28fa 100644 --- a/docs/flatland.rst +++ b/docs/flatland.rst @@ -8,6 +8,7 @@ Subpackages flatland.core flatland.envs + flatland.evaluators flatland.utils Submodules diff --git a/docs/index.rst b/docs/index.rst index 47201ffd35914052e3fde0a72a5d393f33c419e1..08158b557b337cd30515fc34eb9b0a58ecfe4684 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -9,6 +9,7 @@ Welcome to flatland's documentation! installation about_flatland gettingstarted + localevaluation modules FAQ contributing diff --git a/docs/localevaluation.rst b/docs/localevaluation.rst new file mode 100644 index 0000000000000000000000000000000000000000..abfab39aaa33e836981d8e890bd1a9070d3dcae0 --- /dev/null +++ b/docs/localevaluation.rst @@ -0,0 +1,66 @@ +===== +Local Evaluation +===== + +This document explains you how to locally evaluate your submissions before making +an official submission to the competition. + +Requirements +-------------- + +* **flatland-rl** : We expect that you have `flatland-rl` installed by following the instructions in :doc:`installation`. + +* **redis** : Additionally you will also need to have `redis installed <https://redis.io/topics/quickstart>`_ and **should have it running in the background.** + +Test Data +-------------- + +* **test env data** : You can `download and untar the test-env-data <https://www.aicrowd.com/challenges/flatland-challenge/dataset_files>`_, +at a location of your choice, lets say `/path/to/test-env-data/`. After untarring the folder, the folder structure should look something like : + + +.. code-block:: console + + . + └── test-env-data + ├── Test_0 + │  ├── Level_0.pkl + │  └── Level_1.pkl + ├── Test_1 + │  ├── Level_0.pkl + │  └── Level_1.pkl + ├.................. + ├.................. + ├── Test_8 + │  ├── Level_0.pkl + │  └── Level_1.pkl + └── Test_9 + ├── Level_0.pkl + └── Level_1.pkl + +Evaluation Service +------------------ + +* **start evaluation service** : Then you can start the evaluator by running : + +.. code-block:: console + + flatland-evaluator --tests /path/to/test-env-data/ + +RemoteClient +------------------ + +* **run client** : Some `sample submission code can be found in the starter-kit <https://github.com/AIcrowd/flatland-challenge-starter-kit/>`_, +but before you can run your code locally using `FlatlandRemoteClient`, you will have to set the `AICROWD_TESTS_FOLDER` environment variable to the location where you +previous untarred the folder with `the test-env-data`: + +.. code-block:: console + + export AICROWD_TESTS_FOLDER="/path/to/test-env-data/" + + # or on Windows : + # + # set AICROWD_TESTS_FOLDER "\path\to\test-env-data\" + + # and then finally run your code + python run.py