tests/test_random_seeding.py

+import numpy as np
+
+from flatland.envs.observations import GlobalObsForRailEnv, TreeObsForRailEnv
+from flatland.envs.predictions import ShortestPathPredictorForRailEnv
+from flatland.envs.rail_env import RailEnv
+from flatland.envs.rail_generators import rail_from_grid_transition_map, sparse_rail_generator
+from flatland.envs.line_generators import sparse_line_generator
+from flatland.utils.simple_rail import make_simple_rail2
+
+
+def ndom_seeding():
+    # Set fixed malfunction duration for this test
+    rail, rail_map, optionals = make_simple_rail2()
+
+    # Move target to unreachable position in order to not interfere with test
+    for idx in range(100):
+        env = RailEnv(width=25, height=30, rail_generator=rail_from_grid_transition_map(rail, optionals),
+                      line_generator=sparse_line_generator(seed=12), number_of_agents=10)
+        env.reset(True, True, random_seed=1)
+
+        env.agents[0].target = (0, 0)
+        for step in range(10):
+            actions = {}
+            actions[0] = 2
+            env.step(actions)
+        agent_positions = []
+
+        env.agents[0].initial_position == (3, 2)
+        env.agents[1].initial_position == (3, 5)
+        env.agents[2].initial_position == (3, 6)
+        env.agents[3].initial_position == (5, 6)
+        env.agents[4].initial_position == (3, 4)
+        env.agents[5].initial_position == (3, 1)
+        env.agents[6].initial_position == (3, 9)
+        env.agents[7].initial_position == (4, 6)
+        env.agents[8].initial_position == (0, 3)
+        env.agents[9].initial_position == (3, 7)
+        # Test generation print
+        # for a in range(env.get_num_agents()):
+        #    print("env.agents[{}].initial_position == {}".format(a,env.agents[a].initial_position))
+        # print("env.agents[0].initial_position == {}".format(env.agents[0].initial_position))
+        # print("assert env.agents[0].position ==  {}".format(env.agents[0].position))
+
+
+def test_seeding_and_observations():
+    # Test if two different instances diverge with different observations
+    rail, rail_map, optionals = make_simple_rail2()
+    optionals['agents_hints']['num_agents'] = 10
+    # Make two seperate envs with different observation builders
+    # Global Observation
+    env = RailEnv(width=25, height=30, rail_generator=rail_from_grid_transition_map(rail, optionals),
+                  line_generator=sparse_line_generator(seed=12), number_of_agents=10,
+                  obs_builder_object=GlobalObsForRailEnv())
+    # Tree Observation
+    env2 = RailEnv(width=25, height=30, rail_generator=rail_from_grid_transition_map(rail, optionals),
+                   line_generator=sparse_line_generator(seed=12), number_of_agents=10,
+                   obs_builder_object=TreeObsForRailEnv(max_depth=2, predictor=ShortestPathPredictorForRailEnv()))
+
+    env.reset(False, False, random_seed=12)
+    env2.reset(False, False, random_seed=12)
+    # Check that both environments produce the same initial start positions
+    assert env.agents[0].initial_position == env2.agents[0].initial_position
+    assert env.agents[1].initial_position == env2.agents[1].initial_position
+    assert env.agents[2].initial_position == env2.agents[2].initial_position
+    assert env.agents[3].initial_position == env2.agents[3].initial_position
+    assert env.agents[4].initial_position == env2.agents[4].initial_position
+    assert env.agents[5].initial_position == env2.agents[5].initial_position
+    assert env.agents[6].initial_position == env2.agents[6].initial_position
+    assert env.agents[7].initial_position == env2.agents[7].initial_position
+    assert env.agents[8].initial_position == env2.agents[8].initial_position
+    assert env.agents[9].initial_position == env2.agents[9].initial_position
+
+    action_dict = {}
+    for step in range(10):
+        for a in range(env.get_num_agents()):
+            action = np.random.randint(4)
+            action_dict[a] = action
+        env.step(action_dict)
+        env2.step(action_dict)
+    # Check that both environments end up in the same position
+    assert env.agents[0].position == env2.agents[0].position
+    assert env.agents[1].position == env2.agents[1].position
+    assert env.agents[2].position == env2.agents[2].position
+    assert env.agents[3].position == env2.agents[3].position
+    assert env.agents[4].position == env2.agents[4].position
+    assert env.agents[5].position == env2.agents[5].position
+    assert env.agents[6].position == env2.agents[6].position
+    assert env.agents[7].position == env2.agents[7].position
+    assert env.agents[8].position == env2.agents[8].position
+    assert env.agents[9].position == env2.agents[9].position
+    for a in range(env.get_num_agents()):
+        print("assert env.agents[{}].position == env2.agents[{}].position".format(a, a))
+
+
+def test_seeding_and_malfunction():
+    # Test if two different instances diverge with different observations
+    rail, rail_map, optionals = make_simple_rail2()
+    optionals['agents_hints']['num_agents'] = 10
+    stochastic_data = {'prop_malfunction': 0.4,
+                       'malfunction_rate': 2,
+                       'min_duration': 10,
+                       'max_duration': 10}
+    # Make two seperate envs with different and see if the exhibit the same malfunctions
+    # Global Observation
+    for tests in range(1, 100):
+        env = RailEnv(width=25, height=30, rail_generator=rail_from_grid_transition_map(rail, optionals),
+                      line_generator=sparse_line_generator(), number_of_agents=10,
+                      obs_builder_object=GlobalObsForRailEnv())
+
+        # Tree Observation
+        env2 = RailEnv(width=25, height=30, rail_generator=rail_from_grid_transition_map(rail, optionals),
+                       line_generator=sparse_line_generator(), number_of_agents=10,
+                       obs_builder_object=GlobalObsForRailEnv())
+
+        env.reset(True, False, random_seed=tests)
+        env2.reset(True, False, random_seed=tests)
+
+        # Check that both environments produce the same initial start positions
+        assert env.agents[0].initial_position == env2.agents[0].initial_position
+        assert env.agents[1].initial_position == env2.agents[1].initial_position
+        assert env.agents[2].initial_position == env2.agents[2].initial_position
+        assert env.agents[3].initial_position == env2.agents[3].initial_position
+        assert env.agents[4].initial_position == env2.agents[4].initial_position
+        assert env.agents[5].initial_position == env2.agents[5].initial_position
+        assert env.agents[6].initial_position == env2.agents[6].initial_position
+        assert env.agents[7].initial_position == env2.agents[7].initial_position
+        assert env.agents[8].initial_position == env2.agents[8].initial_position
+        assert env.agents[9].initial_position == env2.agents[9].initial_position
+
+        action_dict = {}
+        for step in range(10):
+            for a in range(env.get_num_agents()):
+                action = np.random.randint(4)
+                action_dict[a] = action
+                # print("----------------------")
+                # print(env.agents[a].malfunction_handler, env.agents[a].status)
+                # print(env2.agents[a].malfunction_handler, env2.agents[a].status)
+
+            _, reward1, done1, _ = env.step(action_dict)
+            _, reward2, done2, _ = env2.step(action_dict)
+            for a in range(env.get_num_agents()):
+                assert reward1[a] == reward2[a]
+                assert done1[a] == done2[a]
+        # Check that both environments end up in the same position
+
+        assert env.agents[0].position == env2.agents[0].position
+        assert env.agents[1].position == env2.agents[1].position
+        assert env.agents[2].position == env2.agents[2].position
+        assert env.agents[3].position == env2.agents[3].position
+        assert env.agents[4].position == env2.agents[4].position
+        assert env.agents[5].position == env2.agents[5].position
+        assert env.agents[6].position == env2.agents[6].position
+        assert env.agents[7].position == env2.agents[7].position
+        assert env.agents[8].position == env2.agents[8].position
+        assert env.agents[9].position == env2.agents[9].position
+
+
+def test_reproducability_env():
+    """
+    Test that no random generators are present within the env that get influenced by external np random
+    """
+    speed_ration_map = {1.: 1.,  # Fast passenger train
+                        1. / 2.: 0.,  # Fast freight train
+                        1. / 3.: 0.,  # Slow commuter train
+                        1. / 4.: 0.}  # Slow freight train
+
+    env = RailEnv(width=25, height=30, rail_generator=sparse_rail_generator(max_num_cities=5,
+                                                                            max_rails_between_cities=3,
+                                                                            seed=10,  # Random seed
+                                                                            grid_mode=True
+                                                                            ),
+                  line_generator=sparse_line_generator(speed_ration_map), number_of_agents=1)
+    env.reset(True, True, random_seed=1)
+    excpeted_grid = [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
+                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
+                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
+                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
+                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
+                    [0, 0, 0, 0, 0, 16386, 1025, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
+                    [16386, 17411, 1025, 5633, 17411, 3089, 1025, 1097, 5633, 17411, 1025, 5633, 1025, 1025, 1025, 1025, 5633, 17411, 1025, 1025, 1025, 5633, 17411, 1025, 4608],
+                    [32800, 32800, 0, 72, 3089, 5633, 1025, 17411, 1097, 2064, 0, 72, 1025, 1025, 1025, 1025, 1097, 3089, 1025, 1025, 1025, 1097, 3089, 1025, 37408],
+                    [32800, 32800, 0, 0, 0, 72, 1025, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800],
+                    [32800, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800],
+                    [32800, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800],
+                    [32800, 32872, 4608, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16386, 34864],
+                    [32800, 32800, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800, 32800],
+                    [32800, 32800, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800, 32800],
+                    [32800, 32800, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800, 32800],
+                    [32800, 32800, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800, 32800],
+                    [32800, 32800, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800, 32800],
+                    [32800, 32800, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800, 32800],
+                    [72, 37408, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800, 32800],
+                    [0, 49186, 2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 72, 37408],
+                    [0, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800],
+                    [0, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800],
+                    [0, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800],
+                    [0, 32800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32800],
+                    [0, 32872, 1025, 5633, 17411, 1025, 1025, 1025, 5633, 17411, 1025, 1025, 1025, 1025, 1025, 1025, 5633, 17411, 1025, 1025, 1025, 5633, 17411, 1025, 34864],
+                    [0, 72, 1025, 1097, 3089, 1025, 1025, 1025, 1097, 3089, 1025, 1025, 1025, 1025, 1025, 1025, 1097, 3089, 1025, 1025, 1025, 1097, 3089, 1025, 2064],
+                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
+                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
+                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
+                    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]
+
+    assert env.rail.grid.tolist() == excpeted_grid
+
+    # Test that we don't have interference from calling mulitple function outisde
+    env2 = RailEnv(width=25, height=30, rail_generator=sparse_rail_generator(max_num_cities=5,
+                                                                             max_rails_between_cities=3,
+                                                                             seed=10,  # Random seed
+                                                                             grid_mode=True
+                                                                             ),
+                   line_generator=sparse_line_generator(speed_ration_map), number_of_agents=1)
+    np.random.seed(1)
+    for i in range(10):
+        np.random.randn()
+    env2.reset(True, True, random_seed=1)
+    assert env2.rail.grid.tolist() == excpeted_grid

tests/test_speed_classes.py

+"""Test speed initialization by a map of speeds and their corresponding ratios."""
+import numpy as np
+
+from flatland.envs.rail_env import RailEnv
+from flatland.envs.rail_generators import sparse_rail_generator
+from flatland.envs.line_generators import speed_initialization_helper, sparse_line_generator
+
+
+def test_speed_initialization_helper():
+    random_generator = np.random.RandomState()
+    random_generator.seed(10)
+    speed_ratio_map = {1: 0.3, 2: 0.4, 3: 0.3}
+    actual_speeds = speed_initialization_helper(10, speed_ratio_map, np_random=random_generator)
+
+    # seed makes speed_initialization_helper deterministic -> check generated speeds.
+    assert actual_speeds == [3, 1, 2, 3, 2, 1, 1, 3, 1, 1]
+
+
+def test_rail_env_speed_intializer():
+    speed_ratio_map = {1: 0.3, 2: 0.4, 3: 0.1, 5: 0.2}
+
+    env = RailEnv(width=50, height=50,
+                  rail_generator=sparse_rail_generator(), line_generator=sparse_line_generator(),
+                  number_of_agents=10)
+    env.reset()
+    actual_speeds = list(map(lambda agent: agent.speed_counter.speed, env.agents))
+
+    expected_speed_set = set(speed_ratio_map.keys())
+
+    # check that the number of speeds generated is correct
+    assert len(actual_speeds) == env.get_num_agents()
+
+    # check that only the speeds defined are generated
+    assert all({(actual_speed in expected_speed_set) for actual_speed in actual_speeds})

tests/test_utils.py

+"""Test Utils."""
+from typing import List, Tuple, Optional
+
+import numpy as np
+from attr import attrs, attrib
+
+from flatland.core.grid.grid4 import Grid4TransitionsEnum
+from flatland.envs.agent_utils import EnvAgent
+from flatland.envs.malfunction_generators import MalfunctionParameters, malfunction_from_params
+from flatland.envs.rail_env import RailEnvActions, RailEnv
+from flatland.envs.rail_generators import RailGenerator
+from flatland.envs.line_generators import LineGenerator
+from flatland.utils.rendertools import RenderTool
+from flatland.envs.persistence import RailEnvPersister
+from flatland.envs.step_utils.states import TrainState
+from flatland.envs.step_utils.speed_counter import SpeedCounter
+
+@attrs
+class Replay(object):
+    position = attrib(type=Tuple[int, int])
+    direction = attrib(type=Grid4TransitionsEnum)
+    action = attrib(type=RailEnvActions)
+    malfunction = attrib(default=0, type=int)
+    set_malfunction = attrib(default=None, type=Optional[int])
+    reward = attrib(default=None, type=Optional[float])
+    state = attrib(default=None, type=Optional[TrainState])
+
+
+@attrs
+class ReplayConfig(object):
+    replay = attrib(type=List[Replay])
+    target = attrib(type=Tuple[int, int])
+    speed = attrib(type=float)
+    initial_position = attrib(type=Tuple[int, int])
+    initial_direction = attrib(type=Grid4TransitionsEnum)
+
+
+# ensure that env is working correctly with start/stop/invalidaction penalty different from 0
+def set_penalties_for_replay(env: RailEnv):
+    env.step_penalty = -7
+    env.start_penalty = -13
+    env.stop_penalty = -19
+    env.invalid_action_penalty = -29
+
+
+def run_replay_config(env: RailEnv, test_configs: List[ReplayConfig], rendering: bool = False, activate_agents=True, 
+                      skip_reward_check=False, set_ready_to_depart=False, skip_action_required_check=False):
+    """
+    Runs the replay configs and checks assertions.
+
+    *Initially*
+    - The `initial_position`, `initial_direction`, `target` and `speed` are taken from the `ReplayConfig` to initialize the agents.
+
+    *Before each step*
+    - `position` is verfified
+    - `direction` is verified
+    - `status` is verified (optionally, only if not `None` in `Replay`)
+    - `set_malfunction` is applied (optionally, only if not `None` in `Replay`)
+    - `malfunction` is verified
+    - `action` must only be provided if action_required from previous step (initally all True)
+
+    *Step*
+    - performed with the given `action`
+
+    *After each step*
+    - `reward` is verified after step
+
+
+    Parameters
+    ----------
+    activate_agents: should the agents directly be activated when the environment is initially setup by `reset()`?
+    env: the environment; is `reset()` to set the agents' intial position, direction, target and speed
+    test_configs: the `ReplayConfig`s, one for each agent
+    rendering: should be rendered during replay?
+    """
+    if rendering:
+        renderer = RenderTool(env)
+        renderer.render_env(show=True, frames=False, show_observations=False)
+    info_dict = {
+        'action_required': [True for _ in test_configs]
+    }
+
+    for step in range(len(test_configs[0].replay)):
+        if step == 0:
+            for a, test_config in enumerate(test_configs):
+                agent: EnvAgent = env.agents[a]
+                # set the initial position
+                agent.initial_position = test_config.initial_position
+                agent.initial_direction = test_config.initial_direction
+                agent.direction = test_config.initial_direction
+                agent.target = test_config.target
+                agent.speed_counter = SpeedCounter(speed=test_config.speed)
+            env.reset(False, False)
+
+            if set_ready_to_depart:
+                # Set all agents to ready to depart
+                for i_agent in range(len(env.agents)):
+                    env.agents[i_agent].earliest_departure = 0
+                    env.agents[i_agent]._set_state(TrainState.READY_TO_DEPART)
+
+            elif activate_agents:
+                for a_idx in range(len(env.agents)):
+                    env.agents[a_idx].position =  env.agents[a_idx].initial_position
+                    env.agents[a_idx]._set_state(TrainState.MOVING)
+
+        def _assert(a, actual, expected, msg):
+            print("[{}] verifying {} on agent {}: actual={}, expected={}".format(step, msg, a, actual, expected))
+            assert (actual == expected) or (
+                np.allclose(actual, expected)), "[{}] agent {} {}:  actual={}, expected={}".format(step, a, msg,
+                                                                                                   actual,
+                                                                                                   expected)
+
+        action_dict = {}
+
+        for a, test_config in enumerate(test_configs):
+            agent: EnvAgent = env.agents[a]
+            replay = test_config.replay[step]
+            # if not agent.position == replay.position:
+                # import pdb; pdb.set_trace()   
+            _assert(a, agent.position, replay.position, 'position')
+            _assert(a, agent.direction, replay.direction, 'direction')
+            if replay.state is not None:
+                _assert(a, agent.state, replay.state, 'state')
+
+            if replay.action is not None:
+                if not skip_action_required_check:    
+                    assert info_dict['action_required'][
+                           a] == True or agent.state == TrainState.READY_TO_DEPART, "[{}] agent {} expecting action_required={} or agent status READY_TO_DEPART".format(
+                    step, a, True)
+                action_dict[a] = replay.action
+            else:
+                if not skip_action_required_check:
+                    assert info_dict['action_required'][
+                           a] == False, "[{}] agent {} expecting action_required={}, but found {}".format(
+                    step, a, False, info_dict['action_required'][a])
+
+            if replay.set_malfunction is not None:
+                # As we force malfunctions on the agents we have to set a positive rate that the env
+                # recognizes the agent as potentially malfuncitoning
+                # We also set next malfunction to infitiy to avoid interference with our tests
+                env.agents[a].malfunction_handler._set_malfunction_down_counter(replay.set_malfunction)
+            _assert(a, agent.malfunction_handler.malfunction_down_counter, replay.malfunction, 'malfunction')
+        print(step)
+        _, rewards_dict, _, info_dict = env.step(action_dict)
+        # import pdb; pdb.set_trace()
+        if rendering:
+            renderer.render_env(show=True, show_observations=True)
+
+        for a, test_config in enumerate(test_configs):
+            replay = test_config.replay[step]
+
+            if not skip_reward_check:
+                _assert(a, rewards_dict[a], replay.reward, 'reward')
+
+def create_and_save_env(file_name: str, line_generator: LineGenerator, rail_generator: RailGenerator):
+    stochastic_data = MalfunctionParameters(malfunction_rate=1000,  # Rate of malfunction occurence
+                                            min_duration=15,  # Minimal duration of malfunction
+                                            max_duration=50  # Max duration of malfunction
+                                            )
+
+    env = RailEnv(width=30,
+                  height=30,
+                  rail_generator=rail_generator,
+                  line_generator=line_generator,
+                  number_of_agents=10,
+                  malfunction_generator_and_process_data=malfunction_from_params(stochastic_data),
+                  remove_agents_at_target=True)
+    env.reset(True, True)
+    #env.save(file_name)
+    RailEnvPersister.save(env, file_name)
+    return env

tox.ini

 [tox]
-envlist = py36, py37, examples, notebooks, flake8, docs, coverage, benchmarks
+envlist = py37, py38, examples, docs, coverage
+
 
 [travis]
 python =
+    3.8: py38
     3.7: py37
-    3.6: py36
 
 [flake8]
 max-line-length = 120
 ignore = E121 E126 E123 E128 E133 E226 E241 E242 E704 W291 W293 W391 W503 W504 W505
 
 [testenv:flake8]
-basepython = python
+basepython = python3.7
 passenv = DISPLAY
 deps =
     -r{toxinidir}/requirements_dev.txt
@@ -20,38 +21,52 @@ commands =
     flake8 flatland tests examples benchmarks
 
 [testenv:docs]
-basepython = python
+basepython = python3.7
 whitelist_externals = make
 passenv =
     DISPLAY
     HTTP_PROXY
     HTTPS_PROXY
+conda_deps =
+    tk
+    graphviz
+conda_channels :
+    conda-forge
+    anaconda
 deps =
     -r{toxinidir}/requirements_dev.txt
     -r{toxinidir}/requirements_continuous_integration.txt
+changedir = {toxinidir}
 commands =
     make docs
 
 [testenv:coverage]
-basepython = python
+basepython = python3.7
 whitelist_externals = make
 passenv =
     DISPLAY
 ; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
     HTTP_PROXY
     HTTPS_PROXY
+conda_deps =
+    tk
+conda_channels :
+    conda-forge
+    anaconda
 deps =
     -r{toxinidir}/requirements_dev.txt
     -r{toxinidir}/requirements_continuous_integration.txt
+changedir = {toxinidir}
 commands =
-    make coverage
+    python make_coverage.py
 
 [testenv:benchmarks]
-basepython = python
+basepython = python3.7
 setenv =
     PYTHONPATH = {toxinidir}
 passenv =
     DISPLAY
+    XAUTHORITY
 ; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
     HTTP_PROXY
     HTTPS_PROXY
@@ -59,11 +74,36 @@ whitelist_externals = sh
 deps =
     -r{toxinidir}/requirements_dev.txt
     -r{toxinidir}/requirements_continuous_integration.txt
+changedir = {toxinidir}
 commands =
-    sh -c 'ls benchmarks/*.py  | xargs -n 1 python'
+    python --version
+    python {toxinidir}/benchmarks/benchmark_all_examples.py
+
+[testenv:profiling]
+basepython = python3.7
+setenv =
+    PYTHONPATH = {toxinidir}
+passenv =
+    DISPLAY
+    XAUTHORITY
+; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
+    HTTP_PROXY
+    HTTPS_PROXY
+conda_deps =
+    tk
+conda_channels :
+    conda-forge
+    anaconda
+deps =
+    -r{toxinidir}/requirements_dev.txt
+    -r{toxinidir}/requirements_continuous_integration.txt
+changedir = {toxinidir}
+commands =
+    python {toxinidir}/benchmarks/profile_all_examples.py
 
 [testenv:examples]
-basepython = python
+; TODO should examples be run with py36 and py37??
+basepython = python3.7
 setenv =
     PYTHONPATH = {toxinidir}
 passenv =
@@ -72,19 +112,24 @@ passenv =
 ; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
     HTTP_PROXY
     HTTPS_PROXY
-whitelist_externals = sh
+conda_deps =
+    tk
+conda_channels :
+    conda-forge
+    anaconda
 deps =
     -r{toxinidir}/requirements_dev.txt
+; run tests from subfolder to ensure that resources are accessed via resources and not via relative paths
+changedir = {envtmpdir}/c236d3c240d61a0969d4cb59e2180ce5
 commands =
-    sh -c 'echo DISPLAY=$DISPLAY'
-    sh -c 'echo XAUTHORITY=$XAUTHORITY'
-; pipe echo into python since some examples expect input to close the window after the example is run
-    sh -c 'ls examples/*.py | xargs -I{} -n 1 sh -c "echo -e \"\n====== Running {} ========\n\"; echo | python {}"'
+    python {toxinidir}/benchmarks/run_all_examples.py
 
 [testenv:notebooks]
-basepython = python
+; TODO should examples be run with py36 and py37??
+basepython = python3.7
 setenv =
-    PYTHONPATH = {toxinidir}
+    PYTHONPATH = {envdir}
+    ;{toxinidir}
 passenv =
     DISPLAY
     XAUTHORITY
@@ -92,28 +137,100 @@ passenv =
     HTTP_PROXY
     HTTPS_PROXY
 whitelist_externals = sh
+            bash
+            pwd
 deps =
     -r{toxinidir}/requirements_dev.txt
     -r{toxinidir}/requirements_continuous_integration.txt
+conda_deps =
+    tk
+conda_channels :
+    conda-forge
+    anaconda
+; run tests from subfolder to ensure that resources are accessed via resources and not via relative paths
+changedir = {envtmpdir}/6f59bc68108c3895b1828abdd04b9a06
 commands =
-    sh -c 'jupyter nbextension enable --py --sys-prefix widgetsnbextension'
-    sh -c 'jupyter nbextension enable --py --sys-prefix jpy_canvas'
-; https://stackoverflow.com/questions/35545402/how-to-run-an-ipynb-jupyter-notebook-from-terminal/35545463
-    sh -c 'ls notebooks/*.ipynb  | xargs -n 1 jupyter nbconvert --to python'
-    sh -c 'ls notebooks/*.py  | xargs -I{} -n 1 sh -c "echo -e \"\n====== Running {} ========\n\"; ipython {}"'
+    bash -c "pwd"
+    bash -c "echo $PYTHONPATH"
+    python -m jupyter nbextension install --py --sys-prefix widgetsnbextension
+    python -m jupyter nbextension enable --py --sys-prefix widgetsnbextension
+    python -m jupyter nbextension install --py --sys-prefix jpy_canvas
+    python -m jupyter nbextension enable --py --sys-prefix jpy_canvas
+    python {toxinidir}/notebooks/run_all_notebooks.py
 
-[testenv]
+[testenv:start_jupyter]
+basepython = python3.7
+setenv =
+    PYTHONPATH = {toxinidir}
+passenv =
+    DISPLAY
+    XAUTHORITY
+; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
+    HTTP_PROXY
+    HTTPS_PROXY
 whitelist_externals = sh
-                      pip
+deps =
+    -r{toxinidir}/requirements_dev.txt
+    -r{toxinidir}/requirements_continuous_integration.txt
+conda_deps =
+    tk
+conda_channels :
+    conda-forge
+    anaconda
+changedir = {toxinidir}
+commands =
+    python -m jupyter nbextension install --py --sys-prefix widgetsnbextension
+    python -m jupyter nbextension enable --py --sys-prefix widgetsnbextension
+    python -m jupyter nbextension install --py --sys-prefix jpy_canvas
+    python -m jupyter nbextension enable --py --sys-prefix jpy_canvas
+    python -m jupyter notebook
+
+[testenv:py37]
+platform = linux|linux2|darwin
+basepython = python3.7
+setenv =
+    PYTHONPATH = {toxinidir}
+passenv =
+    DISPLAY
+    XAUTHORITY
+; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
+    HTTP_PROXY
+    HTTPS_PROXY
+conda_deps =
+    tk
+conda_channels :
+    conda-forge
+    anaconda
+deps =
+    -r{toxinidir}/requirements_dev.txt
+; run tests from subfolder to ensure that resources are accessed via resources and not via relative paths
+changedir = {envtmpdir}/fefed3ba12bf1ed81dbcc20fb52706ea
+commands =
+    python --version
+    python -m pytest --basetemp={envtmpdir} {toxinidir}
+
+
+
+[testenv:py38]
+platform = linux|linux2|darwin
+basepython = python3.8
 setenv =
     PYTHONPATH = {toxinidir}
 passenv =
     DISPLAY
+    XAUTHORITY
 ; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
     HTTP_PROXY
     HTTPS_PROXY
+conda_deps =
+    tk
+conda_channels :
+    conda-forge
+    anaconda
 deps =
     -r{toxinidir}/requirements_dev.txt
+; run tests from subfolder to ensure that resources are accessed via resources and not via relative paths
+changedir = {envtmpdir}/fefed3ba12bf1ed81dbcc20fb52706ea
 commands =
-    sh -c 'echo DISPLAY: $DISPLAY'
-    py.test --basetemp={envtmpdir}
+    python --version
+    python -m pytest --basetemp={envtmpdir} {toxinidir}