diff --git a/flatland/envs/rail_env.py b/flatland/envs/rail_env.py
index 7694bb207a0610deb50777346ec4d00fcb55945a..c3ed6e02f80d9d7a6038e03d3984bda0e9afcbeb 100644
--- a/flatland/envs/rail_env.py
+++ b/flatland/envs/rail_env.py
@@ -221,7 +221,7 @@ class RailEnv(Environment):
                 agent.moving = False
                 self.rewards_dict[iAgent] += stop_penalty
 
-            if not agent.moving and action == RailEnvActions.MOVE_FORWARD:
+            if not agent.moving and not (action == RailEnvActions.DO_NOTHING or action == RailEnvActions.STOP_MOVING):
                 # Only allow agent to start moving by pressing forward.
                 agent.moving = True
                 self.rewards_dict[iAgent] += start_penalty