Skip to content
Snippets Groups Projects
Commit e6836f59 authored by hagrid67's avatar hagrid67
Browse files

change to xterm for manual timeout test

parent ffd7a572
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
tests/manual_test_eval_timeout.sh
+ 4
− 2
View file @ e6836f59
......@@ -11,9 +11,11 @@ redis-cli KEYS "*" | grep -i flatland | xargs redis-cli DEL
# you need to create the envs in the folder
# best to delete all but 10 small ones
gnome-terminal --title Service --window -- python -m flatland.evaluators.service --test_folder ../submission-scoring/Envs/neurips2020_round1_v0/
#gnome-terminal --title Service --window -- python -m flatland.evaluators.service --test_folder ../submission-scoring/Envs/neurips2020_round1_v0/
xterm -title Service -hold -e "python -m flatland.evaluators.service --test_folder ../submission-scoring/Envs/neurips2020_round1_v0/ " &
gnome-terminal --title test_eval_timeout --window -- python tests/test_eval_timeout.py
#gnome-terminal --title test_eval_timeout --window -- python tests/test_eval_timeout.py
xterm -title test_eval_timeout -hold -e "python tests/test_eval_timeout.py" &
tests/test_eval_timeout.py
+ 98
− 91
View file @ e6836f59
......@@ -50,7 +50,7 @@ class CustomObservationBuilder(ObservationBuilder):
return self.rail_obs, (status, position, direction, initial_position, target)
remote_client = FlatlandRemoteClient(verbose=False)
def my_controller(obs, number_of_agents):
_action = {}
......@@ -58,102 +58,109 @@ def my_controller(obs, number_of_agents):
_action[_idx] = np.random.randint(0, 5)
return _action
my_observation_builder = CustomObservationBuilder()
evaluation_number = 0
def test_random_timeouts():
remote_client = FlatlandRemoteClient(verbose=False)
step_delay_rate = 0.001
step_delay = 6
my_observation_builder = CustomObservationBuilder()
reset_delay_rate = 0.2
reset_delay = 10
evaluation_number = 0
n_evalations = 10
step_delay_rate = 0.001
step_delay = 6
while True:
reset_delay_rate = 0.2
reset_delay = 10
evaluation_number += 1
# Switch to a new evaluation environemnt
#
# a remote_client.env_create is similar to instantiating a
# RailEnv and then doing a env.reset()
# hence it returns the first observation from the
# env.reset()
#
# You can also pass your custom observation_builder object
# to allow you to have as much control as you wish
# over the observation of your choice.
time_start = time.time()
observation, info = remote_client.env_create(
obs_builder_object=my_observation_builder
)
env_creation_time = time.time() - time_start
if not observation:
#
# If the remote_client returns False on a `env_create` call,
# then it basically means that your agent has already been
# evaluated on all the required evaluation environments,
# and hence its safe to break out of the main evaluation loop
break
print("Evaluation Number : {}".format(evaluation_number))
if np.random.uniform() < reset_delay_rate:
print(f"eval {evaluation_number} sleeping for {reset_delay} seconds")
time.sleep(reset_delay)
local_env = remote_client.env
number_of_agents = len(local_env.agents)
time_taken_by_controller = []
time_taken_per_step = []
steps = 0
while True:
time_start = time.time()
action = my_controller(observation, number_of_agents)
time_taken = time.time() - time_start
time_taken_by_controller.append(time_taken)
while evaluation_number < n_evalations:
evaluation_number += 1
# Switch to a new evaluation environemnt
#
# a remote_client.env_create is similar to instantiating a
# RailEnv and then doing a env.reset()
# hence it returns the first observation from the
# env.reset()
#
# You can also pass your custom observation_builder object
# to allow you to have as much control as you wish
# over the observation of your choice.
time_start = time.time()
try:
observation, all_rewards, done, info = remote_client.env_step(action)
except StopAsyncIteration as err:
print("timeout error ", err)
observation, info = remote_client.env_create(
obs_builder_object=my_observation_builder
)
env_creation_time = time.time() - time_start
if not observation:
#
# If the remote_client returns False on a `env_create` call,
# then it basically means that your agent has already been
# evaluated on all the required evaluation environments,
# and hence its safe to break out of the main evaluation loop
break
steps += 1
time_taken = time.time() - time_start
time_taken_per_step.append(time_taken)
if np.random.uniform() < step_delay_rate:
print(f"step {steps} sleeping for {step_delay} seconds")
time.sleep(step_delay)
if done['__all__']:
print("Reward : ", sum(list(all_rewards.values())))
break
np_time_taken_by_controller = np.array(time_taken_by_controller)
np_time_taken_per_step = np.array(time_taken_per_step)
print("="*100)
print("="*100)
print("Evaluation Number : ", evaluation_number)
print("Current Env Path : ", remote_client.current_env_path)
print("Env Creation Time : ", env_creation_time)
print("Number of Steps : ", steps)
print("Mean/Std of Time taken by Controller : ", np_time_taken_by_controller.mean(), np_time_taken_by_controller.std())
print("Mean/Std of Time per Step : ", np_time_taken_per_step.mean(), np_time_taken_per_step.std())
print("="*100)
print("Evaluation of all environments complete...")
########################################################################
# Submit your Results
#
# Please do not forget to include this call, as this triggers the
# final computation of the score statistics, video generation, etc
# and is necesaary to have your submission marked as successfully evaluated
########################################################################
print(remote_client.submit())
print("Evaluation Number : {}".format(evaluation_number))
if np.random.uniform() < reset_delay_rate:
print(f"eval {evaluation_number} sleeping for {reset_delay} seconds")
time.sleep(reset_delay)
local_env = remote_client.env
number_of_agents = len(local_env.agents)
time_taken_by_controller = []
time_taken_per_step = []
steps = 0
while True:
time_start = time.time()
action = my_controller(observation, number_of_agents)
time_taken = time.time() - time_start
time_taken_by_controller.append(time_taken)
time_start = time.time()
try:
observation, all_rewards, done, info = remote_client.env_step(action)
except StopAsyncIteration as err:
print("timeout error ", err)
break
steps += 1
time_taken = time.time() - time_start
time_taken_per_step.append(time_taken)
if np.random.uniform() < step_delay_rate:
print(f"step {steps} sleeping for {step_delay} seconds")
time.sleep(step_delay)
if done['__all__']:
print("Reward : ", sum(list(all_rewards.values())))
break
np_time_taken_by_controller = np.array(time_taken_by_controller)
np_time_taken_per_step = np.array(time_taken_per_step)
print("="*100)
print("="*100)
print("Evaluation Number : ", evaluation_number)
print("Current Env Path : ", remote_client.current_env_path)
print("Env Creation Time : ", env_creation_time)
print("Number of Steps : ", steps)
print("Mean/Std of Time taken by Controller : ", np_time_taken_by_controller.mean(), np_time_taken_by_controller.std())
print("Mean/Std of Time per Step : ", np_time_taken_per_step.mean(), np_time_taken_per_step.std())
print("="*100)
print("Evaluation of all environments complete...")
########################################################################
# Submit your Results
#
# Please do not forget to include this call, as this triggers the
# final computation of the score statistics, video generation, etc
# and is necesaary to have your submission marked as successfully evaluated
########################################################################
print(remote_client.submit())
if __name__ == "__main__":
test_random_timeouts()
\ No newline at end of file
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment