Jupyter notebooks are ready for flatland3.

But they should be overworked. Might just one example per use case would be much more clear. TODO - cleanup

Jupyter notebooks are ready for flatland3.
But they should be overworked. Might just one example per use case would be much more clear. TODO - cleanup
081108ac · adrian_egli2 · ffac8122 · 081108ac
Commit 081108ac authored 2 years ago by adrian_egli2
--- a/notebooks/test_service.ipynb
+++ b/notebooks/test_service.ipynb
@@ -327,7 +327,6 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "#oFRC = FlatlandRemoteClient(test_envs_root=\"../env_data/tests/service_test/\", verbose=False, use_pickle=True)\n",
    "oFRC = FlatlandRemoteClient(test_env_folder=sDirRoot, verbose=False, use_pickle=True)"
   ]
  },
@@ -337,8 +336,7 @@
   "metadata": {},
   "outputs": [],
   "source": [
-    "#env, env_dict = RailEnvPersister.load_new(\"../env_data/tests/service_test/Test_0/Level_0.pkl\") # env_file)\n",
+    "env, env_dict = RailEnvPersister.load_new(f\"{sDirRoot}/Test_0/Level_0.pkl\")\n",
-    "env, env_dict = RailEnvPersister.load_new(f\"{sDirRoot}/Test_0/Level_0.pkl\") # env_file)\n",
    "ldActions = env_dict[\"actions\"]"
   ]
  },

 %% Cell type:markdown id: tags:
 # Test Service
 Intended to test the service.py evaluator.
 Runs the service.py and a simple client.
 %% Cell type:markdown id: tags:
 # Setup
 %% Cell type:code id: tags:
 ``` python
 %load_ext autoreload
 %autoreload 2
 ```
 %% Cell type:code id: tags:
 ``` python
 import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
 ```
 %% Cell type:code id: tags:
 ``` python
 import PIL
 from flatland.utils.rendertools import RenderTool
 import imageio
 import os
 ```
 %% Cell type:code id: tags:
 ``` python
 from IPython.display import clear_output
 from IPython.core import display
 import ipywidgets as ipw
 display.display(display.HTML("<style>.container { width:95% !important; }</style>"))
 ```
 %% Cell type:code id: tags:
 ``` python
 from flatland.envs.rail_env import RailEnv
 from flatland.envs.rail_generators import sparse_rail_generator
 from flatland.envs.line_generators import sparse_line_generator
 from flatland.envs.malfunction_generators import malfunction_from_file, no_malfunction_generator
 from flatland.envs.rail_generators import rail_from_file
 from flatland.core.env_observation_builder import DummyObservationBuilder
 from flatland.envs.persistence import RailEnvPersister
 from flatland.evaluators.client import FlatlandRemoteClient, TimeoutException
 ```
 %% Cell type:code id: tags:
 ``` python
 import pickle
 import redis
 import subprocess as sp
 import shlex
 import time
 import pkg_resources as pr
 import importlib_resources as ir
 import sys, os
 import pandas as pd
 ```
 %% Cell type:markdown id: tags:
 ### Find the real path of the `env_data` package (should be copied by tox)
 %% Cell type:code id: tags:
 ``` python
 with ir.path("env_data.tests", "test_001.pkl") as oPath:
    sPath = oPath
 print(type(sPath), sPath)
 ```
 %% Cell type:code id: tags:
 ``` python
 sDirRoot = "/" + "/".join(sPath.parts[1:-1] + ("service_test",""))
 sDirRoot
 ```
 %% Cell type:markdown id: tags:
 ### Clear any old redis keys
 %% Cell type:code id: tags:
 ``` python
 oRedis = redis.Redis()
 ```
 %% Cell type:code id: tags:
 ``` python
 lKeys = oRedis.keys("flatland*")
 lKeys
 ```
 %% Cell type:code id: tags:
 ``` python
 for sKey in lKeys:
    print("Deleting:", sKey)
    oRedis.delete(sKey)
 ```
 %% Cell type:markdown id: tags:
 ### Remove `/tmp/output.csv`
 %% Cell type:code id: tags:
 ``` python
 !rm -f /tmp/output.csv
 ```
 %% Cell type:markdown id: tags:
 ### kill any old `service.py` process
 %% Cell type:code id: tags:
 ``` python
 !ps -ef | grep -i python | grep -i flatland.evaluators.service | awk '{print $2}' | xargs kill
 ```
 %% Cell type:code id: tags:
 ``` python
 osEnv2 = os.environ.copy()
 ```
 %% Cell type:markdown id: tags:
 ### Timeouts copied from service.py
 %% Cell type:code id: tags:
 ``` python
 #MAX_SUCCESSIVE_TIMEOUTS = int(os.getenv("FLATLAND_MAX_SUCCESSIVE_TIMEOUTS", 10))
 # 8 hours (will get debug timeout from env variable if applicable)
 #OVERALL_TIMEOUT = int(os.getenv(
 #    "FLATLAND_OVERALL_TIMEOUT",
 #    8 * 60 * 60))
 # 10 mins
 #INTIAL_PLANNING_TIMEOUT = int(os.getenv(
 #    "FLATLAND_INITIAL_PLANNING_TIMEOUT",
 #    10 * 60))
 # 10 seconds
 #PER_STEP_TIMEOUT = int(os.getenv(
 #    "FLATLAND_PER_STEP_TIMEOUT",
 #    10))
 # 5 min - applies to the rest of the commands
 #DEFAULT_COMMAND_TIMEOUT = int(os.getenv(
 #    "FLATLAND_DEFAULT_COMMAND_TIMEOUT",
 #    5 * 60))
 ```
 %% Cell type:markdown id: tags:
 ### Set some short timeouts for testing
 %% Cell type:code id: tags:
 ``` python
 osEnv2["FLATLAND_OVERALL_TIMEOUT"]="10"
 osEnv2["FLATLAND_PER_STEP_TIMEOUT"] = "2"
 osEnv2["FLATLAND_MAX_SUCCESSIVE_TIMEOUTS"] = "2"
 ```
 %% Cell type:markdown id: tags:
 ### Create the python command for `service.py`
 %% Cell type:code id: tags:
 ``` python
 #sCmd = "python -m flatland.evaluators.service --test_folder ../env_data/tests/service_test --mergeDir ./tmp/merge --actionDir ./tmp/actions --pickle --missingOnly"
 #sCmd = "python -m flatland.evaluators.service --test_folder ../env_data/tests/service_test --pickle" # --verbose"
 sCmd = f"python -m flatland.evaluators.service --test_folder {sDirRoot} --pickle" # --verbose"
 lsCmd = shlex.split(sCmd)
 print(sCmd)
 print(lsCmd)
 ```
 %% Cell type:markdown id: tags:
 ### Run the command with Popen (output goes to jupyter stdout not notebook)
 %% Cell type:code id: tags:
 ``` python
 oPipe = sp.Popen(lsCmd, env=osEnv2)
 ```
 %% Cell type:code id: tags:
 ``` python
 oPipe.poll()
 ```
 %% Cell type:code id: tags:
 ``` python
-#oFRC = FlatlandRemoteClient(test_envs_root="../env_data/tests/service_test/", verbose=False, use_pickle=True)
 oFRC = FlatlandRemoteClient(test_env_folder=sDirRoot, verbose=False, use_pickle=True)
 ```
 %% Cell type:code id: tags:
 ``` python
-#env, env_dict = RailEnvPersister.load_new("../env_data/tests/service_test/Test_0/Level_0.pkl") # env_file)
+env, env_dict = RailEnvPersister.load_new(f"{sDirRoot}/Test_0/Level_0.pkl")
-env, env_dict = RailEnvPersister.load_new(f"{sDirRoot}/Test_0/Level_0.pkl") # env_file)
 ldActions = env_dict["actions"]
 ```
 %% Cell type:code id: tags:
 ``` python
 def expert_controller(obs, _env):
    return ldActions[_env._elapsed_steps]
 def random_controller(obs, _env):
    dAct = {}
    for iAg in range(len(_env.agents)):
        dAct[iAg] = np.random.randint(0, 5)
    return dAct
 ```
 %% Cell type:code id: tags:
 ``` python
 oObsB = DummyObservationBuilder()
 ```
 %% Cell type:code id: tags:
 ``` python
 oObsB.get()
 ```
 %% Cell type:code id: tags:
 ``` python
 def run_submission(slow_ep=1, delay=2):
    episode = 0
    obs = True
    while obs:
        obs, info = oFRC.env_create(obs_builder_object=oObsB)
        if not obs:
            print("null observation - all envs completed!")
            break
        print(f"Episode : {episode}")
        print(oFRC.env.dones['__all__'])
        while True:
            if episode < 3:
                action = expert_controller(obs, oFRC.env)
            else:
                action = random_controller(obs, oFRC.env)
            time_start = time.time()
            if (episode == slow_ep) and (oFRC.env._elapsed_steps > 10):
                time.sleep(2)
            try:
                observation, all_rewards, done, info = oFRC.env_step(action)
                time_diff = time.time() - time_start
                print(".", end="")
                if done['__all__']:
                    print("\nCompleted Episode : ", episode)
                    print("Reward : ", sum(list(all_rewards.values())))
                    break
            except TimeoutException as err:
                print("Timeout: ", err)
                break
        episode += 1
    print(f"Evaluation Complete - episodes={episode} - send submit message...")
    print(oFRC.submit())
    print("All done.")
 ```
 %% Cell type:code id: tags:
 ``` python
 try:
    run_submission()
 except TimeoutException as timeoutException:
    print("Timed out.")
    print(timeoutException)
 ```
 %% Cell type:markdown id: tags:
 ### Kill the evaluator process we started earlier
 %% Cell type:code id: tags:
 ``` python
 !ps -ef | grep -i python | grep -i flatland.evaluators.service | awk '{print $2}' | xargs kill
 ```
 %% Cell type:code id: tags:
 ``` python
 pd.read_csv("/tmp/output.csv").T
 ```