Skip to content
Snippets Groups Projects
Commit 7da78512 authored by adrian_egli2's avatar adrian_egli2
Browse files

Notebook is working (Flatland3) - to be tested on build server

parent e7a3322a
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
notebooks/test-service-timeouts.ipynb
+ 2
− 4
View file @ 7da78512
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Test Service # Test Service
Intended to test the service.py evaluator. Intended to test the service.py evaluator.
Runs the service.py and a simple client. Runs the service.py and a simple client.
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
Setup Setup
--- ---
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
%load_ext autoreload %load_ext autoreload
%autoreload 2 %autoreload 2
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import pandas as pd import pandas as pd
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
import PIL import PIL
from flatland.utils.rendertools import RenderTool from flatland.utils.rendertools import RenderTool
import imageio import imageio
import os import os
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
from flatland.envs.rail_env import RailEnv from flatland.envs.rail_env import RailEnv
from flatland.envs.rail_generators import sparse_rail_generator from flatland.envs.rail_generators import sparse_rail_generator
from flatland.envs.schedule_generators import sparse_schedule_generator from flatland.envs.line_generators import sparse_line_generator
from flatland.envs.malfunction_generators import malfunction_from_file, no_malfunction_generator from flatland.envs.malfunction_generators import malfunction_from_file, no_malfunction_generator
from flatland.envs.rail_generators import rail_from_file from flatland.envs.rail_generators import rail_from_file
from flatland.envs.schedule_generators import schedule_from_file
from flatland.core.env_observation_builder import DummyObservationBuilder from flatland.core.env_observation_builder import DummyObservationBuilder
from flatland.envs.persistence import RailEnvPersister from flatland.envs.persistence import RailEnvPersister
from flatland.evaluators.client import FlatlandRemoteClient, TimeoutException from flatland.evaluators.client import FlatlandRemoteClient, TimeoutException
import flatland.evaluators.service as fes
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import pickle import pickle
import redis import redis
import subprocess as sp import subprocess as sp
import shlex import shlex
import time import time
import pkg_resources as pr import pkg_resources as pr
import importlib_resources as ir import importlib_resources as ir
import sys, os import sys, os
import pandas as pd import pandas as pd
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
sDirRoot = "/Users/flaurent/Sites/flatland/neurips2020-flatland-starter-kit/scratch/test-neurips2020-round2-v0" sDirRoot = "/Users/flaurent/Sites/flatland/neurips2020-flatland-starter-kit/scratch/test-neurips2020-round2-v0"
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
!ps -ef | grep -i python | grep -i flatland.evaluators.service !ps -ef | grep -i python | grep -i flatland.evaluators.service
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def kill_evaluator(): def kill_evaluator():
# kill previous evaluator # kill previous evaluator
!ps -ef | grep -i python | grep -i flatland.evaluators.service | awk '{print $2}' | xargs kill !ps -ef | grep -i python | grep -i flatland.evaluators.service | awk '{print $2}' | xargs kill
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def cleanup(): def cleanup():
oRedis = redis.Redis() oRedis = redis.Redis()
lKeys = oRedis.keys("flatland*") lKeys = oRedis.keys("flatland*")
for sKey in lKeys: for sKey in lKeys:
print("Deleting:", sKey) print("Deleting:", sKey)
oRedis.delete(sKey) oRedis.delete(sKey)
!rm -f /tmp/output.csv !rm -f /tmp/output.csv
kill_evaluator() kill_evaluator()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def configure_env(overall=8*60*60, planning=5*60, step=10, successive=10, mean_percentage=0.25): def configure_env(overall=8*60*60, planning=5*60, step=10, successive=10, mean_percentage=0.25):
osEnv2 = os.environ.copy() osEnv2 = os.environ.copy()
osEnv2["FLATLAND_OVERALL_TIMEOUT"]=str(overall) osEnv2["FLATLAND_OVERALL_TIMEOUT"]=str(overall)
osEnv2["FLATLAND_PER_STEP_TIMEOUT"] = str(step) osEnv2["FLATLAND_PER_STEP_TIMEOUT"] = str(step)
osEnv2["FLATLAND_INITIAL_PLANNING_TIMEOUT"] = str(planning) osEnv2["FLATLAND_INITIAL_PLANNING_TIMEOUT"] = str(planning)
osEnv2["FLATLAND_MAX_SUCCESSIVE_TIMEOUTS"] = str(successive) osEnv2["FLATLAND_MAX_SUCCESSIVE_TIMEOUTS"] = str(successive)
osEnv2["TEST_MIN_PERCENTAGE_COMPLETE_MEAN"] = str(mean_percentage) osEnv2["TEST_MIN_PERCENTAGE_COMPLETE_MEAN"] = str(mean_percentage)
return osEnv2 return osEnv2
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def start_evaluator(env): def start_evaluator(env):
sCmd = f"python -m flatland.evaluators.service --test_folder {sDirRoot} --pickle" # --verbose" sCmd = f"python -m flatland.evaluators.service --test_folder {sDirRoot} --pickle" # --verbose"
lsCmd = shlex.split(sCmd) lsCmd = shlex.split(sCmd)
print(sCmd) print(sCmd)
oPipe = sp.Popen(lsCmd, env=env) oPipe = sp.Popen(lsCmd, env=env)
oPipe.poll() oPipe.poll()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def start_client(): def start_client():
oFRC = FlatlandRemoteClient(test_envs_root=sDirRoot, verbose=False, use_pickle=True) oFRC = FlatlandRemoteClient(test_envs_root=sDirRoot, verbose=False, use_pickle=True)
env, env_dict = RailEnvPersister.load_new(f"{sDirRoot}/Test_0/Level_0.pkl") env, env_dict = RailEnvPersister.load_new(f"{sDirRoot}/Test_0/Level_0.pkl")
return oFRC return oFRC
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def random_controller(obs, _env): def random_controller(obs, _env):
np.random.seed(0) np.random.seed(0)
dAct = {} dAct = {}
for iAg in range(len(_env.agents)): for iAg in range(len(_env.agents)):
dAct[iAg] = np.random.randint(0, 5) dAct[iAg] = np.random.randint(0, 5)
return dAct return dAct
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
def run_submission(oFRC, slow_ep=-1, debug=False): def run_submission(oFRC, slow_ep=-1, debug=False):
def log(txt, end="\n"): def log(txt, end="\n"):
if debug: print(txt, end) if debug: print(txt, end)
dummy_obs = DummyObservationBuilder() dummy_obs = DummyObservationBuilder()
episode = 0 episode = 0
obs = True obs = True
while obs: while obs:
obs, info = oFRC.env_create(obs_builder_object=dummy_obs) obs, info = oFRC.env_create(obs_builder_object=dummy_obs)
log(oFRC.current_env_path) log(oFRC.current_env_path)
log(f"Episode : {episode}") log(f"Episode : {episode}")
if not obs: if not obs:
log("None observation - all envs completed!") log("None observation - all envs completed!")
break break
while True: while True:
action = random_controller(obs, oFRC.env) action = random_controller(obs, oFRC.env)
if slow_ep != -1: if slow_ep != -1:
# make a specific episode artificially slow # make a specific episode artificially slow
if (episode == slow_ep) and (oFRC.env._elapsed_steps > 10): if (episode == slow_ep) and (oFRC.env._elapsed_steps > 10):
time.sleep(2) time.sleep(2)
try: try:
observation, all_rewards, done, info = oFRC.env_step(action) observation, all_rewards, done, info = oFRC.env_step(action)
#log(".", end="") #log(".", end="")
if done['__all__']: if done['__all__']:
log("\nCompleted Episode : ", episode) log("\nCompleted Episode : ", episode)
log("Reward : ", sum(list(all_rewards.values()))) log("Reward : ", sum(list(all_rewards.values())))
break break
except TimeoutException as err: except TimeoutException as err:
log("Timeout: ", err) log("Timeout: ", err)
break break
episode += 1 episode += 1
#print(f"Evaluation Complete - episodes={episode} - send submit message...") #print(f"Evaluation Complete - episodes={episode} - send submit message...")
#print(oFRC.submit()) #print(oFRC.submit())
#print("All done.") #print("All done.")
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
Tests Tests
--- ---
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
%%time %%time
# Normal submission # Normal submission
cleanup() cleanup()
config = configure_env(overall=8*60*60, planning=5*60, step=10, successive=10, mean_percentage=0.1) config = configure_env(overall=8*60*60, planning=5*60, step=10, successive=10, mean_percentage=0.1)
start_evaluator(config) start_evaluator(config)
client = start_client() client = start_client()
run_submission(client) run_submission(client)
#try: #try:
# run_submission(client) # run_submission(client)
#except Exception as e: #except Exception as e:
# print("== EXCEPTION! ===") # print("== EXCEPTION! ===")
# print(e) # print(e)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
%%time %%time
# Overall timeout # Overall timeout
cleanup() cleanup()
config = configure_env(overall=3, planning=5*60, step=10, successive=10, mean_percentage=0.0) config = configure_env(overall=3, planning=5*60, step=10, successive=10, mean_percentage=0.0)
start_evaluator(config) start_evaluator(config)
client = start_client() client = start_client()
run_submission(client, debug=False) run_submission(client, debug=False)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
%%time %%time
# Step timeout # Step timeout
# FIXME fails! # FIXME fails!
""" """
cleanup() cleanup()
config = configure_env(overall=3, planning=5*60, step=10, successive=10, mean_percentage=0.0) config = configure_env(overall=3, planning=5*60, step=10, successive=10, mean_percentage=0.0)
start_evaluator(config) start_evaluator(config)
client = start_client() client = start_client()
run_submission(client, slow_ep=2, debug=True) run_submission(client, slow_ep=2, debug=True)
""" """
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
#pd.read_csv("/tmp/output.csv").T #pd.read_csv("/tmp/output.csv").T
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
Cleanup Cleanup
--- ---
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
kill_evaluator() kill_evaluator()
``` ```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment