From 6e07fec539c085992e8b590296a3a7abd33bf236 Mon Sep 17 00:00:00 2001
From: SP Mohanty <spmohanty91@gmail.com>
Date: Thu, 25 Jul 2019 04:40:16 +0200
Subject: [PATCH] Add docs for local evaluation

---
 docs/conf.py                 |  2 +-
 docs/flatland.evaluators.rst | 46 +++++++++++++++++++++++++
 docs/flatland.rst            |  1 +
 docs/index.rst               |  1 +
 docs/localevaluation.rst     | 66 ++++++++++++++++++++++++++++++++++++
 5 files changed, 115 insertions(+), 1 deletion(-)
 create mode 100644 docs/flatland.evaluators.rst
 create mode 100644 docs/localevaluation.rst

diff --git a/docs/conf.py b/docs/conf.py
index 8223e1c7..2ec79aa8 100755
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -33,7 +33,7 @@ sys.path.insert(0, os.path.abspath('..'))
 
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
diff --git a/docs/flatland.evaluators.rst b/docs/flatland.evaluators.rst
new file mode 100644
index 00000000..022e8586
--- /dev/null
+++ b/docs/flatland.evaluators.rst
@@ -0,0 +1,46 @@
+flatland.evaluators package
+===========================
+
+Submodules
+----------
+
+flatland.evaluators.aicrowd\_helpers module
+-------------------------------------------
+
+.. automodule:: flatland.evaluators.aicrowd_helpers
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+flatland.evaluators.client module
+---------------------------------
+
+.. automodule:: flatland.evaluators.client
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+flatland.evaluators.messages module
+-----------------------------------
+
+.. automodule:: flatland.evaluators.messages
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+flatland.evaluators.service module
+----------------------------------
+
+.. automodule:: flatland.evaluators.service
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
+
+Module contents
+---------------
+
+.. automodule:: flatland.evaluators
+    :members:
+    :undoc-members:
+    :show-inheritance:
diff --git a/docs/flatland.rst b/docs/flatland.rst
index fce09a2c..88e8ec93 100644
--- a/docs/flatland.rst
+++ b/docs/flatland.rst
@@ -8,6 +8,7 @@ Subpackages
 
     flatland.core
     flatland.envs
+    flatland.evaluators
     flatland.utils
 
 Submodules
diff --git a/docs/index.rst b/docs/index.rst
index 47201ffd..08158b55 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -9,6 +9,7 @@ Welcome to flatland's documentation!
    installation
    about_flatland
    gettingstarted
+   localevaluation
    modules
    FAQ
    contributing
diff --git a/docs/localevaluation.rst b/docs/localevaluation.rst
new file mode 100644
index 00000000..abfab39a
--- /dev/null
+++ b/docs/localevaluation.rst
@@ -0,0 +1,66 @@
+=====
+Local Evaluation
+=====
+
+This document explains you how to locally evaluate your submissions before making 
+an official submission to the competition.
+
+Requirements
+--------------
+
+* **flatland-rl** : We expect that you have `flatland-rl` installed by following the instructions in  :doc:`installation`.
+
+* **redis** : Additionally you will also need to have  `redis installed <https://redis.io/topics/quickstart>`_ and **should have it running in the background.**
+
+Test Data
+--------------
+
+* **test env data** : You can `download and untar the test-env-data <https://www.aicrowd.com/challenges/flatland-challenge/dataset_files>`_, 
+at a location of your choice, lets say `/path/to/test-env-data/`. After untarring the folder, the folder structure should look something like : 
+
+
+.. code-block:: console
+
+    .
+    └── test-env-data
+        ├── Test_0
+        │   ├── Level_0.pkl
+        │   └── Level_1.pkl
+        ├── Test_1
+        │   ├── Level_0.pkl
+        │   └── Level_1.pkl
+        ├..................
+        ├..................
+        ├── Test_8
+        │   ├── Level_0.pkl
+        │   └── Level_1.pkl
+        └── Test_9
+            ├── Level_0.pkl
+            └── Level_1.pkl
+
+Evaluation Service
+------------------
+
+* **start evaluation service** : Then you can start the evaluator by running : 
+
+.. code-block:: console
+
+    flatland-evaluator --tests /path/to/test-env-data/
+
+RemoteClient
+------------------
+
+* **run client** : Some `sample submission code can be found in the starter-kit <https://github.com/AIcrowd/flatland-challenge-starter-kit/>`_, 
+but before you can run your code locally using `FlatlandRemoteClient`, you will have to set the `AICROWD_TESTS_FOLDER` environment variable to the location where you 
+previous untarred the folder with `the test-env-data`:
+
+.. code-block:: console
+
+    export AICROWD_TESTS_FOLDER="/path/to/test-env-data/"
+
+    # or on Windows :
+    # 
+    # set AICROWD_TESTS_FOLDER "\path\to\test-env-data\"
+
+    # and then finally run your code
+    python run.py
-- 
GitLab