From 2e04d9b7e5d3a3ac4cf2c06497369dd596028cbe Mon Sep 17 00:00:00 2001
From: hagrid67 <jdhwatson@gmail.com>
Date: Sat, 3 Oct 2020 14:57:48 +0100
Subject: [PATCH] minor updates to test-service notebook and service.py.
Changed per-step timeout to 2 seconds, hope to resolve sporadic build
pipeline probs
---
flatland/evaluators/service.py | 10 ---------
notebooks/test-service.ipynb | 40 +++++++++++++++++++---------------
2 files changed, 23 insertions(+), 27 deletions(-)
diff --git a/flatland/evaluators/service.py b/flatland/evaluators/service.py
index ad14ffa8..d77f8cbb 100644
--- a/flatland/evaluators/service.py
+++ b/flatland/evaluators/service.py
@@ -417,10 +417,6 @@ class FlatlandRemoteEvaluationService:
last_simulation_env_file_path
]
-
-
-
-
# Add controller_inference_time_metrics
# These metrics may be missing if no step was done before the episode finished
@@ -448,10 +444,6 @@ class FlatlandRemoteEvaluationService:
self.nb_deadlocked_trains[-1]
] + lrStats
- print(len(lsFields), len(loValues))
- print(lsFields)
- print(loValues)
-
# update the dataframe without the updating-a-copy warning
df = self.evaluation_metadata_df
df.loc[last_simulation_env_file_path, lsFields] = loValues
@@ -463,8 +455,6 @@ class FlatlandRemoteEvaluationService:
#_row.simulation_time = self.simulation_times[-1]
#_row.nb_malfunctioning_trains = self.nb_malfunctioning_trains[-1]
-
-
#_row.controller_inference_time_min = self.stats[
# "current_episode_controller_inference_time_min"
#]
diff --git a/notebooks/test-service.ipynb b/notebooks/test-service.ipynb
index 7d3a09c2..821cb63a 100644
--- a/notebooks/test-service.ipynb
+++ b/notebooks/test-service.ipynb
@@ -341,7 +341,7 @@
"outputs": [],
"source": [
"osEnv2[\"FLATLAND_OVERALL_TIMEOUT\"]=\"10\"\n",
- "osEnv2[\"FLATLAND_PER_STEP_TIMEOUT\"] = \"1\"\n",
+ "osEnv2[\"FLATLAND_PER_STEP_TIMEOUT\"] = \"2\"\n",
"osEnv2[\"FLATLAND_MAX_SUCCESSIVE_TIMEOUTS\"] = \"2\""
]
},
@@ -546,10 +546,10 @@
"====================================================================================================\n",
"## Client Performance Stats\n",
"====================================================================================================\n",
- "\t - env_creation_wait_time\t => min: 0.0006320476531982422 || mean: 0.01032869021097819 || max: 0.026048660278320312\n",
- "\t - internal_env_reset_time\t => min: 0.0017096996307373047 || mean: 0.00208437442779541 || max: 0.0024590492248535156\n",
- "\t - inference_time(approx)\t => min: 2.4318695068359375e-05 || mean: 0.010987052500573663 || max: 2.0010597705841064\n",
- "\t - internal_env_step_time\t => min: 0.00031685829162597656 || mean: 0.000881081099038596 || max: 0.003511667251586914\n",
+ "\t - env_creation_wait_time\t => min: 0.0015025138854980469 || mean: 0.004898786544799805 || max: 0.008767366409301758\n",
+ "\t - internal_env_reset_time\t => min: 0.0014307498931884766 || mean: 0.0017570257186889648 || max: 0.002083301544189453\n",
+ "\t - inference_time(approx)\t => min: 2.2172927856445312e-05 || mean: 0.010976547751921773 || max: 2.0020840167999268\n",
+ "\t - internal_env_step_time\t => min: 0.0003039836883544922 || mean: 0.0007923906976050074 || max: 0.0018360614776611328\n",
"====================================================================================================\n",
"{'mean_reward': 1978.0, 'mean_normalized_reward': 0.40367, 'mean_percentage_complete': 0.5}\n",
"All done.\n"
@@ -714,8 +714,8 @@
" </tr>\n",
" <tr>\n",
" <th>simulation_time</th>\n",
- " <td>0.219365</td>\n",
- " <td>2.04248</td>\n",
+ " <td>0.189325</td>\n",
+ " <td>2.03193</td>\n",
" </tr>\n",
" <tr>\n",
" <th>nb_malfunctioning_trains</th>\n",
@@ -723,19 +723,24 @@
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
+ " <th>nb_deadlocked_trains</th>\n",
+ " <td>0</td>\n",
+ " <td>0</td>\n",
+ " </tr>\n",
+ " <tr>\n",
" <th>controller_inference_time_min</th>\n",
- " <td>2.43187e-05</td>\n",
- " <td>2.43187e-05</td>\n",
+ " <td>2.21729e-05</td>\n",
+ " <td>2.21729e-05</td>\n",
" </tr>\n",
" <tr>\n",
" <th>controller_inference_time_mean</th>\n",
- " <td>5.34392e-05</td>\n",
- " <td>5.2587e-05</td>\n",
+ " <td>3.48147e-05</td>\n",
+ " <td>3.63968e-05</td>\n",
" </tr>\n",
" <tr>\n",
" <th>controller_inference_time_max</th>\n",
- " <td>0.00029397</td>\n",
- " <td>0.00029397</td>\n",
+ " <td>0.000115633</td>\n",
+ " <td>0.000169754</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
@@ -763,11 +768,12 @@
"normalized_reward 0.807347 0\n",
"percentage_complete 1 0\n",
"steps 171 12\n",
- "simulation_time 0.219365 2.04248\n",
+ "simulation_time 0.189325 2.03193\n",
"nb_malfunctioning_trains 2 0\n",
- "controller_inference_time_min 2.43187e-05 2.43187e-05\n",
- "controller_inference_time_mean 5.34392e-05 5.2587e-05\n",
- "controller_inference_time_max 0.00029397 0.00029397"
+ "nb_deadlocked_trains 0 0\n",
+ "controller_inference_time_min 2.21729e-05 2.21729e-05\n",
+ "controller_inference_time_mean 3.48147e-05 3.63968e-05\n",
+ "controller_inference_time_max 0.000115633 0.000169754"
]
},
"execution_count": 29,
--
GitLab