score_test.py

+import time
+
+import numpy as np
+
+from utils.misc_utils import RandomAgent, run_test
+
+with open('parameters.txt','r') as inf:
+    parameters = eval(inf.read())
+
+# Parameter initialization
+features_per_node = 9
+tree_depth = 3
+nodes = 0
+for i in range(tree_depth + 1):
+    nodes += np.power(4, i)
+state_size = features_per_node * nodes * 2
+action_size = 5
+action_dict = dict()
+nr_trials_per_test = 100
+test_results = []
+test_times = []
+test_dones = []
+agent = RandomAgent(state_size, action_size)
+start_time_scoring = time.time()
+test_idx = 0
+score_board = []
+for test_nr in parameters:
+    current_parameters = parameters[test_nr]
+    test_score, test_dones, test_time = run_test(current_parameters, agent, test_nr=test_idx)
+    print('---------')
+    print(' RESULTS')
+    print('---------')
+    print('{} score was {:.3f} with {:.2f}% environments solved. Test took {} Seconds to complete.\n\n\n'.format(
+        test_nr,
+        np.mean(test_score), np.mean(test_dones) * 100, test_time))
+    test_idx += 1
+    score_board.append([test_score, test_dones, test_times])

scoring/README.md

+# Local Submission Scoring
+
+The files in this repo are supposed to help you score your agents behavior locally.
+
+**WARNING**: This is not the actual submission scoring --> Results will differ from the scores you achieve here. But the scoring setup is very similar to this setup.
+
+**Beta Stage**: The scoring function here is still under development, use with caution.
+
+## Introduction
+This repo contains a very basic setup to test your own agent/algorithm on the Flatland scoring setup.
+The repo contains 3 important files:
+
+- `generate_tests.py` Pre-generates the test files for faster testing
+- `score_tests.py` Scores your agent on the generated test files
+- `show_test.py` Shows samples of the generated test files
+- `parameters.txt` Parameters for generating the test files --> These differ in the challenge submission scoring
+
+To start the scoring of your agent you need to do the following
+
+## Parameters used for Level generation
+
+| Test Nr.  | X-Dim  | Y-Dim  | Nr. Agents  | Random Seed  |
+|:---------:|:------:|:------:|:-----------:|:------------:|
+| Test 0      | 10 | 10 | 1 | 3 |
+| Test 1      | 10 | 10 | 3 | 3 |
+| Test 2      | 10 | 10 | 5 | 3 |
+| Test 3      | 50 | 10 | 10 | 3 |
+| Test 4      | 20 | 50 | 10 | 3 |
+| Test 5      | 20 | 20 | 15 | 3 |
+| Test 6      | 50 | 50 | 10 | 3 |
+| Test 7      | 50 | 50 | 40 | 3 |
+| Test 8      | 100 | 100 | 10 | 3 |
+| Test 9      | 100 | 100 | 50 | 3 |
+
+These can be changed if you like to test your agents behavior on different tests.
+
+## Generate the test files
+To generate the set of test files you just have to run `python generate_tests.py`
+This generates pickle files of the levels to test on and places them in the corresponding folders.
+
+## Run Test
+To run the tests you have to modify the `score_tests.py` file to load your agent and the necessary predictor and observation.
+The following lines have to be replaced by you code:
+
+```
+# Load your agent
+agent = YourAgent
+agent.load(Your_Checkpoint)
+
+# Load the necessary Observation Builder and Predictor
+predictor = ShortestPathPredictorForRailEnv()
+observation_builder = TreeObsForRailEnv(max_depth=tree_depth, predictor=predictor)
+```
+
+The agent and the observation builder as well as an observation wrapper can be passed to the test function like this
+
+```
+test_score, test_dones, test_time = run_test(current_parameters, agent, observation_builder=your_observation_builder,
+                                             observation_wrapper=your_observation_wrapper,
+                                             test_nr=test_nr, nr_trials_per_test=10)
+```
+
+In order to speed up the test time you can limit the number of trials per test (`nr_trials_per_test=10`). After you have made these changes to the file you can run `python score_tests.py` which will produce an output similiar to this:
+
+```
+Running Test_0 with (x_dim,y_dim) = (10,10) and 1 Agents.
+Progress: |********************| 100.0% Complete 
+Test_0 score was -0.380 with 100.00% environments solved. Test took 0.62 Seconds to complete.
+
+Running Test_1 with (x_dim,y_dim) = (10,10) and 3 Agents.
+Progress: |********************| 100.0% Complete 
+Test_1 score was -1.540 with 80.00% environments solved. Test took 2.67 Seconds to complete.
+
+Running Test_2 with (x_dim,y_dim) = (10,10) and 5 Agents.
+Progress: |********************| 100.0% Complete 
+Test_2 score was -2.460 with 80.00% environments solved. Test took 4.48 Seconds to complete.
+
+Running Test_3 with (x_dim,y_dim) = (50,10) and 10 Agents.
+Progress: |**__________________| 10.0% Complete
+```
+
+The score is computed by
+
+```
+score = sum(mean(all_rewards))/max_steps
+```
+which is the sum over all time steps and the mean over all agents of the rewards. We normalize it by the maximum number of allowed steps for a level size. The max number of allowed steps is
+
+```
+max_steps = mult_factor * (env.height+env.width)
+```
+Where the `mult_factor` is a multiplication factor to allow for more time if difficulty is to high.
+
+The number of solved envs is just the percentage of episodes that terminated with all agents done.
+
+How these two numbers are used to define your final score will be posted on the [flatland page](https://www.aicrowd.com/organizers/sbb/challenges/flatland-challenge)

scoring/Tests/Test_0/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_1/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_2/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_3/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_4/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_5/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_6/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_7/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_8/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_9/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/generate_tests.py

+import time
+
+import numpy as np
+
+from utils.misc_utils import create_testfiles
+
+with open('parameters.txt', 'r') as inf:
+    parameters = eval(inf.read())
+
+# Parameter initialization
+features_per_node = 9
+tree_depth = 3
+nodes = 0
+for i in range(tree_depth + 1):
+    nodes += np.power(4, i)
+state_size = features_per_node * nodes * 2
+action_size = 5
+action_dict = dict()
+nr_trials_per_test = 100
+test_idx = 0
+
+for test_nr in parameters:
+    current_parameters = parameters[test_nr]
+    create_testfiles(current_parameters, test_nr, nr_trials_per_test=100)

scoring/parameters.txt

+{'Test_0':[10,10,1,3],
+'Test_1':[10,10,3,3],
+'Test_2':[10,10,5,3],
+'Test_3':[50,10,10,3],
+'Test_4':[20,50,10,3],
+'Test_5':[20,20,15,3],
+'Test_6':[50,50,10,3],
+'Test_7':[50,50,40,3],
+'Test_8':[100,100,10,3],
+'Test_9':[100,100,50,3]
+}
\ No newline at end of file

scoring/score_test.py

+import time
+
+import numpy as np
+import torch
+from flatland.envs.observations import TreeObsForRailEnv
+from flatland.envs.predictions import ShortestPathPredictorForRailEnv
+
+from torch_training.dueling_double_dqn import Agent
+from scoring.utils.misc_utils import run_test
+from utils.observation_utils import normalize_observation
+
+with open('parameters.txt', 'r') as inf:
+    parameters = eval(inf.read())
+
+# Parameter initialization
+features_per_node = 9
+tree_depth = 3
+nodes = 0
+for i in range(tree_depth + 1):
+    nodes += np.power(4, i)
+state_size = features_per_node * nodes
+action_size = 5
+action_dict = dict()
+nr_trials_per_test = 100
+test_results = []
+test_times = []
+test_dones = []
+sequential_agent_test = False
+
+# Load your agent
+agent = Agent(state_size, action_size)
+agent.qnetwork_local.load_state_dict(torch.load('../torch_training/Nets/avoid_checkpoint500.pth'))
+
+# Load the necessary Observation Builder and Predictor
+predictor = ShortestPathPredictorForRailEnv()
+observation_builder = TreeObsForRailEnv(max_depth=tree_depth, predictor=predictor)
+
+start_time_scoring = time.time()
+
+score_board = []
+for test_nr in parameters:
+    current_parameters = parameters[test_nr]
+    test_score, test_dones, test_time = run_test(current_parameters, agent, observation_builder=observation_builder,
+                                                 observation_wrapper=normalize_observation,
+                                                 test_nr=test_nr, nr_trials_per_test=10)
+    print('{} score was {:.3f} with {:.2f}% environments solved. Test took {:.2f} Seconds to complete.\n'.format(
+        test_nr,
+        np.mean(test_score), np.mean(test_dones) * 100, test_time))
+
+    score_board.append([np.mean(test_score), np.mean(test_dones) * 100, test_time])
+print('---------')
+print(' RESULTS')
+print('---------')
+test_idx = 0
+for test_nr in parameters:
+    print('{} score was {:.3f}\twith {:.2f}% environments solved.\tTest took {:.2f} Seconds to complete.'.format(
+        test_nr, score_board[test_idx][0], score_board[test_idx][1], score_board[test_idx][2]))
+    test_idx += 1

scoring/show_tests.py

+import time
+
+import numpy as np
+
+from utils.misc_utils import render_test
+
+with open('parameters.txt','r') as inf:
+    parameters = eval(inf.read())
+
+# Parameter initialization
+features_per_node = 9
+tree_depth = 3
+nodes = 0
+for i in range(tree_depth + 1):
+    nodes += np.power(4, i)
+state_size = features_per_node * nodes * 2
+action_size = 5
+action_dict = dict()
+nr_trials_per_test = 100
+test_idx = 0
+
+for test_nr in parameters:
+    current_parameters = parameters[test_nr]
+    render_test(current_parameters, test_nr, nr_examples=2)
+

scoring/utils/misc_utils.py

+import random
+import time
+
+import numpy as np
+from flatland.envs.observations import TreeObsForRailEnv
+from flatland.envs.predictions import ShortestPathPredictorForRailEnv
+from flatland.envs.rail_env import RailEnv
+from flatland.envs.rail_generators import complex_rail_generator, rail_from_file
+from flatland.envs.schedule_generators import complex_schedule_generator
+from flatland.utils.rendertools import RenderTool
+
+# Time factor to test the max time allowed for an env.
+max_time_factor = 1
+
+
+def printProgressBar(iteration, total, prefix='', suffix='', decimals=1, length=100, fill='*'):
+    """
+    Call in a loop to create terminal progress bar
+    @params:
+        iteration   - Required  : current iteration (Int)
+        total       - Required  : total iterations (Int)
+        prefix      - Optional  : prefix string (Str)
+        suffix      - Optional  : suffix string (Str)
+        decimals    - Optional  : positive number of decimals in percent complete (Int)
+        length      - Optional  : character length of bar (Int)
+        fill        - Optional  : bar fill character (Str)
+    """
+    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
+    filledLength = int(length * iteration // total)
+    bar = fill * filledLength + '_' * (length - filledLength)
+    print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end=" ")
+    # Print New Line on Complete
+    if iteration == total:
+        print('')
+
+
+def run_test(parameters, agent, observation_builder=None, observation_wrapper=None, test_nr=0, nr_trials_per_test=100):
+    # Parameter initialization
+    features_per_node = 9
+    start_time_scoring = time.time()
+    action_dict = dict()
+
+    print('Running {} with (x_dim,y_dim) = ({},{}) and {} Agents.'.format(test_nr, parameters[0], parameters[1],
+                                                                          parameters[2]))
+    if observation_builder == None:
+        print("No observation defined!")
+        return
+    # Reset all measurements
+    test_scores = []
+    test_dones = []
+
+    # Reset environment
+    random.seed(parameters[3])
+    np.random.seed(parameters[3])
+
+    printProgressBar(0, nr_trials_per_test, prefix='Progress:', suffix='Complete', length=20)
+    for trial in range(nr_trials_per_test):
+        # Reset the env
+        file_name = "./Tests/{}/Level_{}.pkl".format(test_nr, trial)
+
+        env = RailEnv(width=3,
+                      height=3,
+                      rail_generator=rail_from_file(file_name),
+                      obs_builder_object=observation_builder,
+                      number_of_agents=1,
+                      )
+
+        obs, info = env.reset()
+
+        if observation_wrapper is not None:
+            for a in range(env.get_num_agents()):
+                obs[a] = observation_wrapper(obs[a])
+
+        # Run episode
+        trial_score = 0
+        max_steps = int(max_time_factor * (env.height + env.width))
+        for step in range(max_steps):
+
+            for a in range(env.get_num_agents()):
+                action = agent.act(obs[a], eps=0)
+                action_dict.update({a: action})
+
+            # Environment step
+            obs, all_rewards, done, _ = env.step(action_dict)
+
+            for a in range(env.get_num_agents()):
+                if observation_wrapper is not None:
+                    obs[a] = observation_wrapper(obs[a])
+                trial_score += np.mean(all_rewards[a])
+
+            if done['__all__']:
+                break
+        test_scores.append(trial_score / max_steps)
+        test_dones.append(done['__all__'])
+        printProgressBar(trial + 1, nr_trials_per_test, prefix='Progress:', suffix='Complete', length=20)
+    end_time_scoring = time.time()
+    tot_test_time = end_time_scoring - start_time_scoring
+    return test_scores, test_dones, tot_test_time
+
+
+def create_testfiles(parameters, test_nr=0, nr_trials_per_test=100):
+    # Parameter initialization
+    print('Creating {} with (x_dim,y_dim) = ({},{}) and {} Agents.'.format(test_nr, parameters[0], parameters[1],
+                                                                           parameters[2]))
+    # Reset environment
+    random.seed(parameters[3])
+    np.random.seed(parameters[3])
+    nr_paths = max(4, parameters[2] + int(0.5 * parameters[2]))
+    min_dist = int(min([parameters[0], parameters[1]]) * 0.75)
+    env = RailEnv(width=parameters[0],
+                  height=parameters[1],
+                  rail_generator=complex_rail_generator(nr_start_goal=nr_paths, nr_extra=5, min_dist=min_dist,
+                                                        max_dist=99999,
+                                                        seed=parameters[3]),
+                  schedule_generator=complex_schedule_generator(),
+                  obs_builder_object=TreeObsForRailEnv(max_depth=2),
+                  number_of_agents=parameters[2])
+    printProgressBar(0, nr_trials_per_test, prefix='Progress:', suffix='Complete', length=20)
+    for trial in range(nr_trials_per_test):
+        # Reset the env
+        env.reset(True, True)
+        env.save("./Tests/{}/Level_{}.pkl".format(test_nr, trial))
+        printProgressBar(trial + 1, nr_trials_per_test, prefix='Progress:', suffix='Complete', length=20)
+
+    return
+
+
+def render_test(parameters, test_nr=0, nr_examples=5):
+    for trial in range(nr_examples):
+        # Reset the env
+        print('Showing {} Level {} with (x_dim,y_dim) = ({},{}) and {} Agents.'.format(test_nr, trial, parameters[0],
+                                                                                       parameters[1],
+                                                                                       parameters[2]))
+        file_name = "./Tests/{}/Level_{}.pkl".format(test_nr, trial)
+
+        env = RailEnv(width=1,
+                      height=1,
+                      rail_generator=rail_from_file(file_name),
+                      obs_builder_object=TreeObsForRailEnv(max_depth=2),
+                      number_of_agents=1,
+                      )
+        env_renderer = RenderTool(env, gl="PILSVG", )
+        env_renderer.set_new_rail()
+
+        env.reset(False, False)
+        env_renderer.render_env(show=True, show_observations=False)
+
+        time.sleep(0.1)
+        env_renderer.close_window()
+    return
+
+
+def run_test_sequential(parameters, agent, test_nr=0, tree_depth=3):
+    # Parameter initialization
+    features_per_node = 9
+    start_time_scoring = time.time()
+    action_dict = dict()
+    nr_trials_per_test = 100
+    print('Running {} with (x_dim,y_dim) = ({},{}) and {} Agents.'.format(test_nr, parameters[0], parameters[1],
+                                                                          parameters[2]))
+
+    # Reset all measurements
+    test_scores = []
+    test_dones = []
+
+    # Reset environment
+    random.seed(parameters[3])
+    np.random.seed(parameters[3])
+
+    printProgressBar(0, nr_trials_per_test, prefix='Progress:', suffix='Complete', length=20)
+    for trial in range(nr_trials_per_test):
+        # Reset the env
+        file_name = "./Tests/{}/Level_{}.pkl".format(test_nr, trial)
+
+        env = RailEnv(width=3,
+                      height=3,
+                      rail_generator=rail_from_file(file_name),
+                      obs_builder_object=TreeObsForRailEnv(max_depth=tree_depth,
+                                                           predictor=ShortestPathPredictorForRailEnv()),
+                      number_of_agents=1,
+                      )
+
+        obs, info = env.reset()
+        done = env.dones
+        # Run episode
+        trial_score = 0
+        max_steps = int(max_time_factor * (env.height + env.width))
+        for step in range(max_steps):
+
+            # Action
+            acting_agent = 0
+            for a in range(env.get_num_agents()):
+                if done[a]:
+                    acting_agent += 1
+                if acting_agent == a:
+                    action = agent.act(obs[acting_agent], eps=0)
+                else:
+                    action = 0
+                action_dict.update({a: action})
+
+            # Environment step
+
+            obs, all_rewards, done, _ = env.step(action_dict)
+            for a in range(env.get_num_agents()):
+                trial_score += np.mean(all_rewards[a])
+            if done['__all__']:
+                break
+        test_scores.append(trial_score / max_steps)
+        test_dones.append(done['__all__'])
+        printProgressBar(trial + 1, nr_trials_per_test, prefix='Progress:', suffix='Complete', length=20)
+    end_time_scoring = time.time()
+    tot_test_time = end_time_scoring - start_time_scoring
+    return test_scores, test_dones, tot_test_time

sequential_agent/run_test.py

+import numpy as np
+from flatland.envs.observations import TreeObsForRailEnv
+from flatland.envs.predictions import ShortestPathPredictorForRailEnv
+from flatland.envs.rail_env import RailEnv
+from flatland.envs.rail_generators import complex_rail_generator
+from flatland.envs.schedule_generators import complex_schedule_generator
+from flatland.utils.rendertools import RenderTool
+
+from sequential_agent.simple_order_agent import OrderedAgent
+
+np.random.seed(2)
+"""
+file_name = "../torch_training/railway/complex_scene.pkl"
+env = RailEnv(width=10,
+              height=20,
+              rail_generator=rail_from_file(file_name),
+              obs_builder_object=TreeObsForRailEnv(max_depth=1, predictor=ShortestPathPredictorForRailEnv()))
+x_dim = env.width
+y_dim = env.height
+
+"""
+
+x_dim = 20  # np.random.randint(8, 20)
+y_dim = 20  # np.random.randint(8, 20)
+n_agents = 10  # np.random.randint(3, 8)
+n_goals = n_agents + np.random.randint(0, 3)
+min_dist = int(0.75 * min(x_dim, y_dim))
+
+env = RailEnv(width=x_dim,
+              height=y_dim,
+              rail_generator=complex_rail_generator(nr_start_goal=n_goals, nr_extra=5, min_dist=min_dist,
+                                                    max_dist=99999,
+                                                    seed=0),
+              schedule_generator=complex_schedule_generator(),
+              obs_builder_object=TreeObsForRailEnv(max_depth=1, predictor=ShortestPathPredictorForRailEnv()),
+              number_of_agents=n_agents)
+env.reset(True, True)
+
+tree_depth = 1
+observation_helper = TreeObsForRailEnv(max_depth=tree_depth, predictor=ShortestPathPredictorForRailEnv())
+env_renderer = RenderTool(env, gl="PILSVG", )
+handle = env.get_agent_handles()
+n_trials = 1
+max_steps = 100 * (env.height + env.width)
+record_images = False
+agent = OrderedAgent()
+action_dict = dict()
+
+for trials in range(1, n_trials + 1):
+
+    # Reset environment
+    obs, info = env.reset(True, True)
+    done = env.dones
+    env_renderer.reset()
+    frame_step = 0
+    # Run episode
+    for step in range(max_steps):
+        env_renderer.render_env(show=True, show_observations=False, show_predictions=True)
+
+        if record_images:
+            env_renderer.gl.save_image("./Images/flatland_frame_{:04d}.bmp".format(frame_step))
+            frame_step += 1
+
+        # Action
+        acting_agent = 0
+        for a in range(env.get_num_agents()):
+            if done[a]:
+                acting_agent += 1
+            if a == acting_agent:
+                action = agent.act(obs[a], eps=0)
+                print(action)
+            else:
+                action = 4
+            action_dict.update({a: action})
+
+        # Environment step
+
+        obs, all_rewards, done, _ = env.step(action_dict)
+
+        if done['__all__']:
+            break