diff --git a/flatland/evaluators/client.py b/flatland/evaluators/client.py
index f74af355d8e6233d7569d9a786068586bbf6a089..cc31caaf1f6d2f1721fac550751b85390842cc4a 100644
--- a/flatland/evaluators/client.py
+++ b/flatland/evaluators/client.py
@@ -46,6 +46,7 @@ class FlatlandRemoteClient(object):
remote_port=6379,
remote_db=0,
remote_password=None,
+ test_envs_root=None,
verbose=False):
self.remote_host = remote_host
@@ -58,14 +59,22 @@ class FlatlandRemoteClient(object):
db=remote_db,
password=remote_password)
self.namespace = "flatland-rl"
- try:
- self.service_id = os.environ['FLATLAND_RL_SERVICE_ID']
- except KeyError:
- self.service_id = "FLATLAND_RL_SERVICE_ID"
+ self.service_id = os.getenv(
+ 'FLATLAND_RL_SERVICE_ID',
+ 'FLATLAND_RL_SERVICE_ID'
+ )
self.command_channel = "{}::{}::commands".format(
self.namespace,
self.service_id
)
+ if test_envs_root:
+ self.test_envs_root = test_envs_root
+ else:
+ self.test_envs_root = os.getenv(
+ 'AICROWD_TESTS_FOLDER',
+ '/tmp/flatland_envs'
+ )
+
self.verbose = verbose
self.env = None
@@ -161,6 +170,12 @@ class FlatlandRemoteClient(object):
return observation
test_env_file_path = _response['payload']['env_file_path']
+ print("Received Env : ", test_env_file_path)
+ test_env_file_path = os.path.join(
+ self.test_envs_root,
+ test_env_file_path
+ )
+ print("Current env path : ", test_env_file_path)
self.env = RailEnv(
width=1,
height=1,
@@ -192,11 +207,15 @@ class FlatlandRemoteClient(object):
remote_info = _payload['info']
# Replicate the action in the local env
- local_observation, local_rewards, local_done, local_info = \
+ local_observation, local_reward, local_done, local_info = \
self.env.step(action)
- assert are_dicts_equal(remote_reward, local_rewards)
- assert are_dicts_equal(remote_done, local_done)
+ print(local_reward)
+ if not are_dicts_equal(remote_reward, local_reward):
+ raise Exception("local and remote `reward` are diverging")
+ print(remote_reward, local_reward)
+ if not are_dicts_equal(remote_done, local_done):
+ raise Exception("local and remote `done` are diverging")
# Return local_observation instead of remote_observation
# as the remote_observation is build using a dummy observation
diff --git a/flatland/evaluators/service.py b/flatland/evaluators/service.py
index 3ef69c7dc1467d1967b1ad0247117038ad027b0c..56fbb7a22705f5bf0d8fc213b5c1ac05cde852d6 100644
--- a/flatland/evaluators/service.py
+++ b/flatland/evaluators/service.py
@@ -61,9 +61,8 @@ class FlatlandRemoteEvaluationService:
# Test Env folder Paths
self.test_env_folder = test_env_folder
self.video_generation_envs = video_generation_envs
- self.video_generation_indices = []
self.env_file_paths = self.get_env_filepaths()
- print(self.video_generation_indices)
+ print(self.env_file_paths)
# Logging and Reporting related vars
self.verbose = verbose
@@ -101,7 +100,7 @@ class FlatlandRemoteEvaluationService:
self.env = False
self.env_renderer = False
self.reward = 0
- self.simulation_count = 0
+ self.simulation_count = -1
self.simulation_rewards = []
self.simulation_percentage_complete = []
self.simulation_steps = []
@@ -151,18 +150,6 @@ class FlatlandRemoteEvaluationService:
x, self.test_env_folder
) for x in env_paths])
- for _idx, env_path in enumerate(env_paths):
- """
- Here we collect the indices of the environments for which
- we need to generate the videos
-
- We increment the simulation count on env_create
- so the 1st simulation has an index of 1, when comparing in
- env_step
- """
- for vg_env in self.video_generation_envs:
- if vg_env in env_path:
- self.video_generation_indices.append(_idx+1)
return env_paths
def instantiate_redis_connection_pool(self):
@@ -283,13 +270,19 @@ class FlatlandRemoteEvaluationService:
Add a high level summary of everything thats
hapenning here.
"""
-
+ self.simulation_count += 1
if self.simulation_count < len(self.env_file_paths):
"""
There are still test envs left that are yet to be evaluated
"""
test_env_file_path = self.env_file_paths[self.simulation_count]
+ print("__ Env Path : ", test_env_file_path)
+ test_env_file_path = os.path.join(
+ self.test_env_folder,
+ test_env_file_path
+ )
+ print("__ Processed Env Path : ", test_env_file_path)
del self.env
self.env = RailEnv(
width=1,
@@ -299,15 +292,13 @@ class FlatlandRemoteEvaluationService:
)
if self.visualize:
if self.env_renderer:
- del self.env_renderer
+ del self.env_renderer
self.env_renderer = RenderTool(self.env, gl="PILSVG", )
# Set max episode steps allowed
self.env._max_episode_steps = \
int(1.5 * (self.env.width + self.env.height))
- self.simulation_count += 1
-
if self.begin_simulation:
# If begin simulation has already been initialized
# atleast once
@@ -326,7 +317,7 @@ class FlatlandRemoteEvaluationService:
_command_response['type'] = messages.FLATLAND_RL.ENV_CREATE_RESPONSE
_command_response['payload'] = {}
_command_response['payload']['observation'] = _observation
- _command_response['payload']['env_file_path'] = test_env_file_path
+ _command_response['payload']['env_file_path'] = self.env_file_paths[self.simulation_count]
else:
"""
All test env evaluations are complete
@@ -389,12 +380,17 @@ class FlatlandRemoteEvaluationService:
# Record Frame
if self.visualize:
- self.env_renderer.render_env(show=False, show_observations=False, show_predictions=False)
+ self.env_renderer.render_env(
+ show=False,
+ show_observations=False,
+ show_predictions=False
+ )
"""
Only save the frames for environments which are separately provided
in video_generation_indices param
"""
- if self.simulation_count in self.video_generation_indices:
+ current_env_path = self.env_file_paths[self.simulation_count]
+ if current_env_path in self.video_generation_envs:
self.env_renderer.gl.save_image(
os.path.join(
self.vizualization_folder_name,