parameters.txt

+{'Test_0':[20,20,20,3],
+'Test_1':[10,10,3,4321],
+'Test_2':[10,10,5,123],
+'Test_3':[50,50,5,21],
+'Test_4':[50,50,20,85],
+'Test_5':[100,100,5,436],
+'Test_6':[100,100,20,6487],
+'Test_7':[100,100,50,567],
+'Test_8':[100,10,20,3245],
+'Test_9':[10,100,20,632]
+}
\ No newline at end of file

requirements_RLLib_training.txt

+#ray==0.7.0
+gym ==0.12.5
+opencv-python==4.1.0.25
+#tensorflow==1.13.1
+lz4==2.1.10
+gin-config==0.1.4
\ No newline at end of file

requirements_torch_training.txt

+git+https://gitlab.aicrowd.com/flatland/flatland.git
+importlib-metadata>=0.17
+importlib_resources>=1.0.2
+torch>=1.1.0
\ No newline at end of file

score_test.py

+import time
+
+import numpy as np
+
+from utils.misc_utils import RandomAgent, run_test
+
+with open('parameters.txt','r') as inf:
+    parameters = eval(inf.read())
+
+# Parameter initialization
+features_per_node = 9
+tree_depth = 3
+nodes = 0
+for i in range(tree_depth + 1):
+    nodes += np.power(4, i)
+state_size = features_per_node * nodes * 2
+action_size = 5
+action_dict = dict()
+nr_trials_per_test = 100
+test_results = []
+test_times = []
+test_dones = []
+agent = RandomAgent(state_size, action_size)
+start_time_scoring = time.time()
+test_idx = 0
+score_board = []
+for test_nr in parameters:
+    current_parameters = parameters[test_nr]
+    test_score, test_dones, test_time = run_test(current_parameters, agent, test_nr=test_idx)
+    print('---------')
+    print(' RESULTS')
+    print('---------')
+    print('{} score was {:.3f} with {:.2f}% environments solved. Test took {} Seconds to complete.\n\n\n'.format(
+        test_nr,
+        np.mean(test_score), np.mean(test_dones) * 100, test_time))
+    test_idx += 1
+    score_board.append([test_score, test_dones, test_times])

scoring/README.md

+# Local Submission Scoring
+
+The files in this repo are supposed to help you score your agents behavior locally.
+
+**WARNING**: This is not the actual submission scoring --> Results will differ from the scores you achieve here. But the scoring setup is very similar to this setup.
+
+**Beta Stage**: The scoring function here is still under development, use with caution.
+
+## Introduction
+This repo contains a very basic setup to test your own agent/algorithm on the Flatland scoring setup.
+The repo contains 3 important files:
+
+- `generate_tests.py` Pre-generates the test files for faster testing
+- `score_tests.py` Scores your agent on the generated test files
+- `show_test.py` Shows samples of the generated test files
+- `parameters.txt` Parameters for generating the test files --> These differ in the challenge submission scoring
+
+To start the scoring of your agent you need to do the following
+
+## Parameters used for Level generation
+
+| Test Nr.  | X-Dim  | Y-Dim  | Nr. Agents  | Random Seed  |
+|:---------:|:------:|:------:|:-----------:|:------------:|
+| Test 0      | 10 | 10 | 1 | 3 |
+| Test 1      | 10 | 10 | 3 | 3 |
+| Test 2      | 10 | 10 | 5 | 3 |
+| Test 3      | 50 | 10 | 10 | 3 |
+| Test 4      | 20 | 50 | 10 | 3 |
+| Test 5      | 20 | 20 | 15 | 3 |
+| Test 6      | 50 | 50 | 10 | 3 |
+| Test 7      | 50 | 50 | 40 | 3 |
+| Test 8      | 100 | 100 | 10 | 3 |
+| Test 9      | 100 | 100 | 50 | 3 |
+
+These can be changed if you like to test your agents behavior on different tests.
+
+## Generate the test files
+To generate the set of test files you just have to run `python generate_tests.py`
+This generates pickle files of the levels to test on and places them in the corresponding folders.
+
+## Run Test
+To run the tests you have to modify the `score_tests.py` file to load your agent and the necessary predictor and observation.
+The following lines have to be replaced by you code:
+
+```
+# Load your agent
+agent = YourAgent
+agent.load(Your_Checkpoint)
+
+# Load the necessary Observation Builder and Predictor
+predictor = ShortestPathPredictorForRailEnv()
+observation_builder = TreeObsForRailEnv(max_depth=tree_depth, predictor=predictor)
+```
+
+The agent and the observation builder as well as an observation wrapper can be passed to the test function like this
+
+```
+test_score, test_dones, test_time = run_test(current_parameters, agent, observation_builder=your_observation_builder,
+                                             observation_wrapper=your_observation_wrapper,
+                                             test_nr=test_nr, nr_trials_per_test=10)
+```
+
+In order to speed up the test time you can limit the number of trials per test (`nr_trials_per_test=10`). After you have made these changes to the file you can run `python score_tests.py` which will produce an output similiar to this:
+
+```
+Running Test_0 with (x_dim,y_dim) = (10,10) and 1 Agents.
+Progress: |********************| 100.0% Complete 
+Test_0 score was -0.380 with 100.00% environments solved. Test took 0.62 Seconds to complete.
+
+Running Test_1 with (x_dim,y_dim) = (10,10) and 3 Agents.
+Progress: |********************| 100.0% Complete 
+Test_1 score was -1.540 with 80.00% environments solved. Test took 2.67 Seconds to complete.
+
+Running Test_2 with (x_dim,y_dim) = (10,10) and 5 Agents.
+Progress: |********************| 100.0% Complete 
+Test_2 score was -2.460 with 80.00% environments solved. Test took 4.48 Seconds to complete.
+
+Running Test_3 with (x_dim,y_dim) = (50,10) and 10 Agents.
+Progress: |**__________________| 10.0% Complete
+```
+
+The score is computed by
+
+```
+score = sum(mean(all_rewards))/max_steps
+```
+which is the sum over all time steps and the mean over all agents of the rewards. We normalize it by the maximum number of allowed steps for a level size. The max number of allowed steps is
+
+```
+max_steps = mult_factor * (env.height+env.width)
+```
+Where the `mult_factor` is a multiplication factor to allow for more time if difficulty is to high.
+
+The number of solved envs is just the percentage of episodes that terminated with all agents done.
+
+How these two numbers are used to define your final score will be posted on the [flatland page](https://www.aicrowd.com/organizers/sbb/challenges/flatland-challenge)

scoring/Tests/Test_0/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_1/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_2/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_3/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_4/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_5/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_6/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_7/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_8/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/Tests/Test_9/.gitignore

+## Ignore everything in this directory
+*
+# Except this file
+!.gitignore
\ No newline at end of file

scoring/generate_tests.py

+import time
+
+import numpy as np
+
+from utils.misc_utils import create_testfiles
+
+with open('parameters.txt', 'r') as inf:
+    parameters = eval(inf.read())
+
+# Parameter initialization
+features_per_node = 9
+tree_depth = 3
+nodes = 0
+for i in range(tree_depth + 1):
+    nodes += np.power(4, i)
+state_size = features_per_node * nodes * 2
+action_size = 5
+action_dict = dict()
+nr_trials_per_test = 100
+test_idx = 0
+
+for test_nr in parameters:
+    current_parameters = parameters[test_nr]
+    create_testfiles(current_parameters, test_nr, nr_trials_per_test=100)

scoring/parameters.txt

+{'Test_0':[10,10,1,3],
+'Test_1':[10,10,3,3],
+'Test_2':[10,10,5,3],
+'Test_3':[50,10,10,3],
+'Test_4':[20,50,10,3],
+'Test_5':[20,20,15,3],
+'Test_6':[50,50,10,3],
+'Test_7':[50,50,40,3],
+'Test_8':[100,100,10,3],
+'Test_9':[100,100,50,3]
+}
\ No newline at end of file

scoring/score_test.py

+import time
+
+import numpy as np
+import torch
+from flatland.envs.observations import TreeObsForRailEnv
+from flatland.envs.predictions import ShortestPathPredictorForRailEnv
+
+from torch_training.dueling_double_dqn import Agent
+from scoring.utils.misc_utils import run_test
+from utils.observation_utils import normalize_observation
+
+with open('parameters.txt', 'r') as inf:
+    parameters = eval(inf.read())
+
+# Parameter initialization
+features_per_node = 9
+tree_depth = 3
+nodes = 0
+for i in range(tree_depth + 1):
+    nodes += np.power(4, i)
+state_size = features_per_node * nodes
+action_size = 5
+action_dict = dict()
+nr_trials_per_test = 100
+test_results = []
+test_times = []
+test_dones = []
+sequential_agent_test = False
+
+# Load your agent
+agent = Agent(state_size, action_size)
+agent.qnetwork_local.load_state_dict(torch.load('../torch_training/Nets/avoid_checkpoint500.pth'))
+
+# Load the necessary Observation Builder and Predictor
+predictor = ShortestPathPredictorForRailEnv()
+observation_builder = TreeObsForRailEnv(max_depth=tree_depth, predictor=predictor)
+
+start_time_scoring = time.time()
+
+score_board = []
+for test_nr in parameters:
+    current_parameters = parameters[test_nr]
+    test_score, test_dones, test_time = run_test(current_parameters, agent, observation_builder=observation_builder,
+                                                 observation_wrapper=normalize_observation,
+                                                 test_nr=test_nr, nr_trials_per_test=10)
+    print('{} score was {:.3f} with {:.2f}% environments solved. Test took {:.2f} Seconds to complete.\n'.format(
+        test_nr,
+        np.mean(test_score), np.mean(test_dones) * 100, test_time))
+
+    score_board.append([np.mean(test_score), np.mean(test_dones) * 100, test_time])
+print('---------')
+print(' RESULTS')
+print('---------')
+test_idx = 0
+for test_nr in parameters:
+    print('{} score was {:.3f}\twith {:.2f}% environments solved.\tTest took {:.2f} Seconds to complete.'.format(
+        test_nr, score_board[test_idx][0], score_board[test_idx][1], score_board[test_idx][2]))
+    test_idx += 1

scoring/show_tests.py

+import time
+
+import numpy as np
+
+from utils.misc_utils import render_test
+
+with open('parameters.txt','r') as inf:
+    parameters = eval(inf.read())
+
+# Parameter initialization
+features_per_node = 9
+tree_depth = 3
+nodes = 0
+for i in range(tree_depth + 1):
+    nodes += np.power(4, i)
+state_size = features_per_node * nodes * 2
+action_size = 5
+action_dict = dict()
+nr_trials_per_test = 100
+test_idx = 0
+
+for test_nr in parameters:
+    current_parameters = parameters[test_nr]
+    render_test(current_parameters, test_nr, nr_examples=2)
+