From b049ff5293a33be7c4d1ae51237a872cfdd393ca Mon Sep 17 00:00:00 2001
From: hagrid67 <jdhwatson@gmail.com>
Date: Wed, 8 Jul 2020 22:17:58 +0100
Subject: [PATCH] adding manual script to run test for eval timeout
---
flatland/evaluators/service.py | 30 ++++++++++--------------------
tests/manual_test_eval_timeout.sh | 19 +++++++++++++++++++
2 files changed, 29 insertions(+), 20 deletions(-)
create mode 100755 tests/manual_test_eval_timeout.sh
diff --git a/flatland/evaluators/service.py b/flatland/evaluators/service.py
index 21cbd684..0ce65ccd 100644
--- a/flatland/evaluators/service.py
+++ b/flatland/evaluators/service.py
@@ -522,8 +522,8 @@ class FlatlandRemoteEvaluationService:
_redis = self.get_redis_connection()
#command_response_channel = command['response_channel']
- if self.verbose and not suppress_logs:
- print("Responding with : ", error_dict)
+ #if self.verbose and not suppress_logs:
+ print("Send error : ", error_dict)
if self.use_pickle:
sResponse = pickle.dumps(error_dict)
@@ -757,21 +757,6 @@ class FlatlandRemoteEvaluationService:
))
self.record_frame_step += 1
- def send_env_step_timeout(self, command):
- print("handle_env_step_timeout")
- error_dict = dict(
- type=messages.FLATLAND_RL.ENV_STEP_TIMEOUT,
-
- # payload probably unnecessary
- payload=dict(
- observation=False,
- env_file_path=False,
- info=False,
- random_seed=False
- ))
-
- self.send_error(error_dict)
-
def save_actions(self):
sfEnv = self.env_file_paths[self.simulation_count]
@@ -995,11 +980,16 @@ class FlatlandRemoteEvaluationService:
try:
command = self.get_next_command()
+
except timeout_decorator.timeout_decorator.TimeoutError:
if self.previous_command['type'] == messages.FLATLAND_RL.ENV_STEP:
- self.send_env_step_timeout({"error":messages.FLATLAND_RL.ENV_STEP_TIMEOUT})
- self.state_env_timed_out = True
- continue
+ self.send_error({"type":messages.FLATLAND_RL.ENV_STEP_TIMEOUT})
+
+ elif self.previous_command['type'] == messages.FLATLAND_RL.ENV_CREATE:
+ self.send_error({"type":messages.FLATLAND_RL.ENV_RESET_TIMEOUT})
+
+ self.state_env_timed_out = True
+ continue
if "timestamp" in command.keys():
latency = time.time() - command["timestamp"]
diff --git a/tests/manual_test_eval_timeout.sh b/tests/manual_test_eval_timeout.sh
new file mode 100755
index 00000000..b527b4f2
--- /dev/null
+++ b/tests/manual_test_eval_timeout.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+export FLATLAND_DEFAULT_COMMAND_TIMEOUT=5
+export FLATLAND_DEFAULT_COMMAND_TIMEOUT=60
+export FLATLAND_INITIAL_PLANNING_TIMEOUT=8
+
+export AICROWD_TESTS_FOLDER=../submission-scoring/Envs/neurips2020_round1_v0
+
+redis-cli KEYS "*" | grep -i flatland | xargs redis-cli DEL
+
+
+# you need to create the envs in the folder
+# best to delete all but 10 small ones
+gnome-terminal --window -- python -m flatland.evaluators.service --test_folder ../submission-scoring/Envs/neurips2020_round1_v0/
+
+gnome-terminal --window -- python tests/test_eval_timeout.py
+
+
+
--
GitLab