diff --git a/flatland/evaluators/service.py b/flatland/evaluators/service.py
index 09b98b4d9d0081b3da4651e0142f103383df1f8b..064471535e89d6fe87c16ecefac9127bbf8c6ffa 100644
--- a/flatland/evaluators/service.py
+++ b/flatland/evaluators/service.py
@@ -284,7 +284,7 @@ class FlatlandRemoteEvaluationService:
             self.evaluation_metadata_df["controller_inference_time_mean"] = np.nan
             self.evaluation_metadata_df["controller_inference_time_max"] = np.nan
         else:
-            print("[WARNING] metadata.csv not found in tests folder. Granular metric collection is hence Disabled.")
+            raise Exception("metadata.csv not found in tests folder. Please use an updated version of the test set.")
 
     def update_evaluation_metadata(self):
         """