change to xterm for manual timeout test

tests/manual_test_eval_timeout.sh

@@ -11,9 +11,11 @@ redis-cli KEYS "*" | grep -i flatland | xargs redis-cli DEL
 
 # you need to create the envs in the folder
 # best to delete all but 10 small ones
-gnome-terminal --title Service --window -- python -m flatland.evaluators.service --test_folder  ../submission-scoring/Envs/neurips2020_round1_v0/   
+#gnome-terminal --title Service --window -- python -m flatland.evaluators.service --test_folder  ../submission-scoring/Envs/neurips2020_round1_v0/   
+xterm -title Service -hold -e "python -m flatland.evaluators.service --test_folder  ../submission-scoring/Envs/neurips2020_round1_v0/   " &
 
-gnome-terminal --title test_eval_timeout  --window -- python tests/test_eval_timeout.py
+#gnome-terminal --title test_eval_timeout  --window -- python tests/test_eval_timeout.py
+xterm -title test_eval_timeout  -hold -e "python tests/test_eval_timeout.py" &
 
 
 

tests/test_eval_timeout.py

@@ -50,7 +50,7 @@ class CustomObservationBuilder(ObservationBuilder):
 
         return self.rail_obs, (status, position, direction, initial_position, target)
 
-remote_client = FlatlandRemoteClient(verbose=False)
+
 
 def my_controller(obs, number_of_agents):
     _action = {}
@@ -58,102 +58,109 @@ def my_controller(obs, number_of_agents):
         _action[_idx] = np.random.randint(0, 5)
     return _action
 
-my_observation_builder = CustomObservationBuilder()
 
-evaluation_number = 0
+def test_random_timeouts():
+    remote_client = FlatlandRemoteClient(verbose=False)
 
-step_delay_rate = 0.001
-step_delay = 6
+    my_observation_builder = CustomObservationBuilder()
 
-reset_delay_rate = 0.2
-reset_delay = 10
+    evaluation_number = 0
+    n_evalations = 10
 
+    step_delay_rate = 0.001
+    step_delay = 6
 
-while True:
+    reset_delay_rate = 0.2
+    reset_delay = 10
 
-    evaluation_number += 1
-    # Switch to a new evaluation environemnt
-    # 
-    # a remote_client.env_create is similar to instantiating a 
-    # RailEnv and then doing a env.reset()
-    # hence it returns the first observation from the 
-    # env.reset()
-    # 
-    # You can also pass your custom observation_builder object
-    # to allow you to have as much control as you wish 
-    # over the observation of your choice.
-    time_start = time.time()
-    observation, info = remote_client.env_create(
-                    obs_builder_object=my_observation_builder
-                )
-    env_creation_time = time.time() - time_start
-    if not observation:
-        #
-        # If the remote_client returns False on a `env_create` call,
-        # then it basically means that your agent has already been 
-        # evaluated on all the required evaluation environments,
-        # and hence its safe to break out of the main evaluation loop
-        break
-    
-    print("Evaluation Number : {}".format(evaluation_number))
-
-    if np.random.uniform() < reset_delay_rate:
-        print(f"eval {evaluation_number} sleeping for {reset_delay} seconds")
-        time.sleep(reset_delay)
-
-    local_env = remote_client.env
-    number_of_agents = len(local_env.agents)
-
-    time_taken_by_controller = []
-    time_taken_per_step = []
-    steps = 0
-
-
-
-    while True:
-        time_start = time.time()
-        action = my_controller(observation, number_of_agents)
-        time_taken = time.time() - time_start
-        time_taken_by_controller.append(time_taken)
+    while evaluation_number < n_evalations:
 
+        evaluation_number += 1
+        # Switch to a new evaluation environemnt
+        # 
+        # a remote_client.env_create is similar to instantiating a 
+        # RailEnv and then doing a env.reset()
+        # hence it returns the first observation from the 
+        # env.reset()
+        # 
+        # You can also pass your custom observation_builder object
+        # to allow you to have as much control as you wish 
+        # over the observation of your choice.
         time_start = time.time()
-
-        try:
-            observation, all_rewards, done, info = remote_client.env_step(action)
-        except StopAsyncIteration as err:
-            print("timeout error ", err)
+        observation, info = remote_client.env_create(
+                        obs_builder_object=my_observation_builder
+                    )
+        env_creation_time = time.time() - time_start
+        if not observation:
+            #
+            # If the remote_client returns False on a `env_create` call,
+            # then it basically means that your agent has already been 
+            # evaluated on all the required evaluation environments,
+            # and hence its safe to break out of the main evaluation loop
             break
-
-        steps += 1
-        time_taken = time.time() - time_start
-        time_taken_per_step.append(time_taken)
-
-        if np.random.uniform() < step_delay_rate:
-            print(f"step {steps} sleeping for {step_delay} seconds")
-            time.sleep(step_delay)
-
-        if done['__all__']:
-            print("Reward : ", sum(list(all_rewards.values())))
-            break
-    
-    np_time_taken_by_controller = np.array(time_taken_by_controller)
-    np_time_taken_per_step = np.array(time_taken_per_step)
-    print("="*100)
-    print("="*100)
-    print("Evaluation Number : ", evaluation_number)
-    print("Current Env Path : ", remote_client.current_env_path)
-    print("Env Creation Time : ", env_creation_time)
-    print("Number of Steps : ", steps)
-    print("Mean/Std of Time taken by Controller : ", np_time_taken_by_controller.mean(), np_time_taken_by_controller.std())
-    print("Mean/Std of Time per Step : ", np_time_taken_per_step.mean(), np_time_taken_per_step.std())
-    print("="*100)
-
-print("Evaluation of all environments complete...")
-########################################################################
-# Submit your Results
-# 
-# Please do not forget to include this call, as this triggers the 
-# final computation of the score statistics, video generation, etc
-# and is necesaary to have your submission marked as successfully evaluated
-########################################################################
-print(remote_client.submit())
+        
+        print("Evaluation Number : {}".format(evaluation_number))
+
+        if np.random.uniform() < reset_delay_rate:
+            print(f"eval {evaluation_number} sleeping for {reset_delay} seconds")
+            time.sleep(reset_delay)
+
+        local_env = remote_client.env
+        number_of_agents = len(local_env.agents)
+
+        time_taken_by_controller = []
+        time_taken_per_step = []
+        steps = 0
+
+
+
+        while True:
+            time_start = time.time()
+            action = my_controller(observation, number_of_agents)
+            time_taken = time.time() - time_start
+            time_taken_by_controller.append(time_taken)
+
+            time_start = time.time()
+
+            try:
+                observation, all_rewards, done, info = remote_client.env_step(action)
+            except StopAsyncIteration as err:
+                print("timeout error ", err)
+                break
+
+            steps += 1
+            time_taken = time.time() - time_start
+            time_taken_per_step.append(time_taken)
+
+            if np.random.uniform() < step_delay_rate:
+                print(f"step {steps} sleeping for {step_delay} seconds")
+                time.sleep(step_delay)
+
+            if done['__all__']:
+                print("Reward : ", sum(list(all_rewards.values())))
+                break
+        
+        np_time_taken_by_controller = np.array(time_taken_by_controller)
+        np_time_taken_per_step = np.array(time_taken_per_step)
+        print("="*100)
+        print("="*100)
+        print("Evaluation Number : ", evaluation_number)
+        print("Current Env Path : ", remote_client.current_env_path)
+        print("Env Creation Time : ", env_creation_time)
+        print("Number of Steps : ", steps)
+        print("Mean/Std of Time taken by Controller : ", np_time_taken_by_controller.mean(), np_time_taken_by_controller.std())
+        print("Mean/Std of Time per Step : ", np_time_taken_per_step.mean(), np_time_taken_per_step.std())
+        print("="*100)
+
+    print("Evaluation of all environments complete...")
+    ########################################################################
+    # Submit your Results
+    # 
+    # Please do not forget to include this call, as this triggers the 
+    # final computation of the score statistics, video generation, etc
+    # and is necesaary to have your submission marked as successfully evaluated
+    ########################################################################
+    print(remote_client.submit())
+
+if __name__ == "__main__":
+    test_random_timeouts()
\ No newline at end of file