Skip to content
Snippets Groups Projects
Commit 060968e3 authored by hagrid67's avatar hagrid67
Browse files

use importlib_resources in test_service notebook - find path to env_data

parent 89dd3132
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
notebooks/notebook-list
+ 1
− 0
View file @ 060968e3
......@@ -9,3 +9,4 @@ simple_example_3_manual_control.ipynb
Simple_Rendering_Demo.ipynb
test-collision.ipynb
test-saved-envs.ipynb
test-service.ipynb
notebooks/test-service.ipynb
+ 105
− 43
View file @ 060968e3
%% Cell type:markdown id: tags:
# Test Service
Intended to test the service.py evaluator.
Runs the service.py and a simple client.
%% Cell type:markdown id: tags:
# Setup
%% Cell type:code id: tags:
``` python
%load_ext autoreload
%autoreload 2
```
%% Cell type:code id: tags:
``` python
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
```
%% Cell type:code id: tags:
``` python
import PIL
from flatland.utils.rendertools import RenderTool
import imageio
import os
```
%% Cell type:code id: tags:
``` python
from IPython.display import clear_output
from IPython.core import display
import ipywidgets as ipw
display.display(display.HTML("<style>.container { width:95% !important; }</style>"))
```
%% Output
%% Cell type:code id: tags:
``` python
#sPack, sResource = "env_data.tests", "Test_2_Level_0.pkl"
sPack, sResource = "env_data.tests", "Test_9_Level_1.pkl"
```
%% Cell type:code id: tags:
``` python
import pickle
from flatland.envs.rail_env import RailEnv
from flatland.envs.rail_generators import sparse_rail_generator
from flatland.envs.schedule_generators import sparse_schedule_generator
from flatland.envs.malfunction_generators import malfunction_from_file, no_malfunction_generator
from flatland.envs.rail_generators import rail_from_file
from flatland.envs.schedule_generators import schedule_from_file
from flatland.core.env_observation_builder import DummyObservationBuilder
```
%% Cell type:code id: tags:
``` python
from flatland.envs.persistence import RailEnvPersister
from flatland.evaluators.client import FlatlandRemoteClient
import redis
import subprocess as sp
import shlex
import time
import pkg_resources as pr
import importlib_resources as ir
import sys
```
%% Cell type:code id: tags:
``` python
!pwd
```
%% Output
/home/jeremy/projects/aicrowd/rl-trains/flatland5/notebooks
%% Cell type:markdown id: tags:
### Find the real path of the `env_data` package (should be copied by tox)
%% Cell type:code id: tags:
``` python
with ir.path("env_data.tests", "test_001.pkl") as oPath:
sPath = oPath
print(type(sPath), sPath)
```
%% Output
<class 'pathlib.PosixPath'> /home3/jeremy/projects/aicrowd/rl-trains/flatland5/env_data/tests/test_001.pkl
%% Cell type:code id: tags:
``` python
sDirRoot = "/" + "/".join(sPath.parts[1:-1] + ("service_test",""))
sDirRoot
```
%% Output
'/home3/jeremy/projects/aicrowd/rl-trains/flatland5/env_data/tests/service_test/'
%% Cell type:markdown id: tags:
### Clear any old redis keys
%% Cell type:code id: tags:
``` python
oRedis = redis.Redis()
```
%% Cell type:code id: tags:
``` python
lKeys = oRedis.keys("flatland*")
lKeys
```
%% Output
[]
[b'flatland-rl::FLATLAND_RL_SERVICE_ID::response::65c5cdafbda515c05db3af5b2c7800ce']
%% Cell type:code id: tags:
``` python
for sKey in lKeys:
print("Deleting:", sKey)
oRedis.delete(sKey)
```
%% Output
Deleting: b'flatland-rl::FLATLAND_RL_SERVICE_ID::response::65c5cdafbda515c05db3af5b2c7800ce'
%% Cell type:markdown id: tags:
## Service python command
### Kill any old service process
%% Cell type:code id: tags:
``` python
!ps -ef | grep -i python | grep -i flatland.evaluators.service | awk '{print $2}' | xargs kill
```
%% Cell type:code id: tags:
``` python
#sCmd = "python -m flatland.evaluators.service --test_folder ../env_data/tests/service_test --mergeDir ./tmp/merge --actionDir ./tmp/actions --pickle --missingOnly"
sCmd = "python -m flatland.evaluators.service --test_folder ../env_data/tests/service_test --pickle" # --verbose"
#sCmd = "python -m flatland.evaluators.service --test_folder ../env_data/tests/service_test --pickle" # --verbose"
sCmd = f"python -m flatland.evaluators.service --test_folder {sDirRoot} --pickle" # --verbose"
lsCmd = shlex.split(sCmd)
print(sCmd)
print(lsCmd)
```
%% Output
python -m flatland.evaluators.service --test_folder /home3/jeremy/projects/aicrowd/rl-trains/flatland5/env_data/tests/service_test/ --pickle
['python', '-m', 'flatland.evaluators.service', '--test_folder', '/home3/jeremy/projects/aicrowd/rl-trains/flatland5/env_data/tests/service_test/', '--pickle']
%% Cell type:code id: tags:
``` python
#wOut = ipw.Output()
#wOut
```
%% Cell type:code id: tags:
``` python
oPipe = sp.Popen(lsCmd)
```
%% Cell type:code id: tags:
``` python
type(oPipe)
```
%% Output
subprocess.Popen
%% Cell type:code id: tags:
``` python
oPipe.poll()
```
%% Cell type:code id: tags:
``` python
oFRC = FlatlandRemoteClient(test_envs_root="../env_data/tests/service_test/", verbose=False, use_pickle=True)
#oFRC = FlatlandRemoteClient(test_envs_root="../env_data/tests/service_test/", verbose=False, use_pickle=True)
oFRC = FlatlandRemoteClient(test_envs_root=sDirRoot, verbose=False, use_pickle=True)
```
%% Cell type:code id: tags:
``` python
env, env_dict = RailEnvPersister.load_new("../env_data/tests/service_test/Test_0/Level_0.pkl") # env_file)
#env, env_dict = RailEnvPersister.load_new("../env_data/tests/service_test/Test_0/Level_0.pkl") # env_file)
env, env_dict = RailEnvPersister.load_new(f"{sDirRoot}/Test_0/Level_0.pkl") # env_file)
ldActions = env_dict["actions"]
```
%% Cell type:code id: tags:
``` python
def expert_controller(obs, _env):
return ldActions[_env._elapsed_steps]
def random_controller(obs, _env):
dAct = {}
for iAg in range(len(_env.agents)):
dAct[iAg] = np.random.randint(0, 5)
return dAct
```
%% Cell type:code id: tags:
``` python
oObsB = DummyObservationBuilder()
```
%% Cell type:code id: tags:
``` python
oObsB.get()
```
%% Output
True
%% Cell type:code id: tags:
``` python
if True:
episode = 0
obs = True
while obs:
obs, info = oFRC.env_create(
obs_builder_object=oObsB
)
if not obs:
print("null observation!")
"""
The remote env returns False as the first obs
when it is done evaluating all the individual episodes
"""
break
print("Episode : {}".format(episode))
episode += 1
print(oFRC.env.dones['__all__'])
while True:
if episode < 3:
action = expert_controller(obs, oFRC.env)
else:
action = random_controller(obs, oFRC.env)
time_start = time.time()
try:
observation, all_rewards, done, info = oFRC.env_step(action)
time_diff = time.time() - time_start
#print("Step Time : ", time_diff)
print(".", end="")
if done['__all__']:
print("\nCurrent Episode : ", episode)
print("Episode Done")
print("Reward : ", sum(list(all_rewards.values())))
break
except TimeoutException as err:
print("Timeout: ", err)
break
print("Evaluation Complete...")
print(oFRC.submit())
```
%% Output
DEPRECATED - use FileMalfunctionGen instead of malfunction_from_file
DEPRECATED - RailEnv arg: malfunction_and_process_data - use malfunction_generator
Episode : 0
False
...........................................................................................................................................................................
Current Episode : 1
Episode Done
Reward : 10.0
DEPRECATED - use FileMalfunctionGen instead of malfunction_from_file
DEPRECATED - RailEnv arg: malfunction_and_process_data - use malfunction_generator
Episode : 1
False
...........................................................................................................................................................................
Current Episode : 2
Episode Done
Reward : 10.0
null observation!
Evaluation Complete...
====================================================================================================
====================================================================================================
## Client Performance Stats
====================================================================================================
- env_creation_wait_time => min: 0.0011365413665771484 || mean: 0.007825930913289389 || max: 0.014734029769897461
- internal_env_reset_time => min: 0.0015034675598144531 || mean: 0.0029495954513549805 || max: 0.004395723342895508
- inference_time(approx) => min: 2.288818359375e-05 || mean: 6.703675141808588e-05 || max: 0.0007417201995849609
- internal_env_step_time => min: 0.0003027915954589844 || mean: 0.0009439960557814924 || max: 0.0038805007934570312
- env_creation_wait_time => min: 0.0010077953338623047 || mean: 0.0071858565012613935 || max: 0.014672040939331055
- internal_env_reset_time => min: 0.002426624298095703 || mean: 0.0024870634078979492 || max: 0.0025475025177001953
- inference_time(approx) => min: 2.1696090698242188e-05 || mean: 4.437164953577589e-05 || max: 0.0003075599670410156
- internal_env_step_time => min: 0.00030541419982910156 || mean: 0.0008465407187478588 || max: 0.0026504993438720703
====================================================================================================
{'mean_reward': -944.0, 'mean_normalized_reward': 0.80735, 'mean_percentage_complete': 1.0}
%% Cell type:markdown id: tags:
### Kill the evaluator process we started earlier
%% Cell type:code id: tags:
``` python
!ps -ef | grep -i python | grep -i flatland.evaluators.service | awk '{print $2}' | xargs kill
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment