diff --git a/flatland/evaluators/client.py b/flatland/evaluators/client.py index c813223b792f57dbe2329b887db77c7cc284f19d..545d52a7d5ad11cf951de1de34fb62ae953fa4ff 100644 --- a/flatland/evaluators/client.py +++ b/flatland/evaluators/client.py @@ -253,15 +253,16 @@ class FlatlandRemoteClient(object): def env_step(self, action, render=False): """ Respond with [observation, reward, done, info] - """ + """ + # We use the last_env_step_time as an approximate measure of the inference time + approximate_inference_time = time.time() - self.last_env_step_time + self.update_running_stats("inference_time(approx)", approximate_inference_time) + _request = {} _request['type'] = messages.FLATLAND_RL.ENV_STEP _request['payload'] = {} _request['payload']['action'] = action - - # We use the last_env_step_time as an approximate measure of the inference time - approximate_inference_time = time.time() - self.last_env_step_time - self.update_running_stats("inference_time(approx)", approximate_inference_time) + _request['payload']['inference_time'] = approximate_inference_time # Relay the action in a non-blocking way to the server # so that it can start doing an env.step on it in ~ parallel diff --git a/flatland/evaluators/service.py b/flatland/evaluators/service.py index 4849b5f604b9bb90784c7376807575b14b2804f2..129d1ebddc91df9dfdc715a90a1523d293c27ce3 100644 --- a/flatland/evaluators/service.py +++ b/flatland/evaluators/service.py @@ -196,7 +196,6 @@ class FlatlandRemoteEvaluationService: self.stats[max_key] = scalar self.stats[counter_key] = 1 - def get_env_filepaths(self): """ Gathers a list of all available rail env files to be used @@ -246,8 +245,9 @@ class FlatlandRemoteEvaluationService: ) if os.path.exists(metadata_file_path): self.evaluation_metadata_df = pd.read_csv(metadata_file_path) - self.evaluation_metadata_df["filename"] = self.evaluation_metadata_df["test_id"] + \ - "/" + self.evaluation_metadata_df["env_id"] + ".pkl" + self.evaluation_metadata_df["filename"] = \ + self.evaluation_metadata_df["test_id"] + \ + "/" + self.evaluation_metadata_df["env_id"] + ".pkl" self.evaluation_metadata_df = self.evaluation_metadata_df.set_index("filename") # Add custom columns @@ -458,7 +458,6 @@ class FlatlandRemoteEvaluationService: malfunction_generator_and_process_data=malfunction_from_file(test_env_file_path), obs_builder_object=DummyObservationBuilder()) - if self.begin_simulation: # If begin simulation has already been initialized # atleast once @@ -548,6 +547,9 @@ class FlatlandRemoteEvaluationService: has done['__all__']==True") action = _payload['action'] + inference_time = _payload['inference_time'] + self.update_running_stats("controller_inference_time", inference_time) + time_start = time.time() _observation, all_rewards, done, info = self.env.step(action) time_diff = time.time() - time_start @@ -625,7 +627,6 @@ class FlatlandRemoteEvaluationService: self.stats[max_key])) print("=" * 100) - # Register simulation time of the last episode self.simulation_times.append(time.time() - self.begin_simulation) # Compute the evaluation metadata for the last episode