Merge branch 'fix-benchmarks' into 'master'

Fix benchmarks

See merge request flatland/flatland!91

.gitlab-ci.yml

@@ -54,8 +54,6 @@ benchmarks_and_profiling:
         - tests
     stage: benchmarks_and_profiling
     only:
-#        refs:
-#            - master
         variables:
             - $BENCHMARKS_AND_PROFILING
     script:

benchmarks/benchmark_all_examples.py

@@ -9,6 +9,10 @@ import pkg_resources
 from benchmarker import Benchmarker
 from importlib_resources import path
 
+from flatland.utils import graphics_pil
+
+graphics_pil.unattended_switch = True
+
 for entry in [entry for entry in importlib_resources.contents('examples') if
               not pkg_resources.resource_isdir('examples', entry)
               and entry.endswith(".py")

benchmarks/benchmark_utils.py

+import contextlib
+
+
+# Copied from https://docs.python.org/3/library/test.html
+
+@contextlib.contextmanager
+def swap_attr(obj, attr, new_val):
+    """Temporary swap out an attribute with a new object.
+
+    Usage:
+        with swap_attr(obj, "attr", 5):
+            ...
+
+        This will set obj.attr to 5 for the duration of the with: block,
+        restoring the old value at the end of the block. If `attr` doesn't
+        exist on `obj`, it will be created and then deleted at the end of the
+        block.
+
+        The old value (or None if it doesn't exist) will be assigned to the
+        target of the "as" clause, if there is one.
+    """
+    if hasattr(obj, attr):
+        real_val = getattr(obj, attr)
+        setattr(obj, attr, new_val)
+        try:
+            yield real_val
+        finally:
+            setattr(obj, attr, real_val)
+    else:
+        setattr(obj, attr, new_val)
+        try:
+            yield
+        finally:
+            if hasattr(obj, attr):
+                delattr(obj, attr)

benchmarks/profile_all_examples.py

@@ -2,19 +2,17 @@ import cProfile
 import runpy
 import sys
 from io import StringIO
-from test.support import swap_attr
 
 import importlib_resources
 import pkg_resources
 from importlib_resources import path
 
+from benchmarks.benchmark_utils import swap_attr
+from flatland.utils import graphics_pil
+
 
 def profile(resource, entry):
     with path(resource, entry) as file_in:
-        # we use the test package, which is meant for internal use by Python only internal and
-        # Any use of this package outside of Python’s standard library is discouraged as code (..)
-        # can change or be removed without notice between releases of Python.
-        # https://docs.python.org/3/library/test.html
         # TODO remove input() from examples
         print("*****************************************************************")
         print("Profiling {}".format(entry))
@@ -27,6 +25,8 @@ def profile(resource, entry):
             cProfile.run('my_func()', sort='time')
 
 
+graphics_pil.unattended_switch = True
+
 for entry in [entry for entry in importlib_resources.contents('examples') if
               not pkg_resources.resource_isdir('examples', entry)
               and entry.endswith(".py")

benchmarks/run_all_examples.py

+import runpy
+import sys
+from io import StringIO
+
+import importlib_resources
+import pkg_resources
+from importlib_resources import path
+
+from benchmarks.benchmark_utils import swap_attr
+from flatland.utils import graphics_pil
+
+graphics_pil.unattended_switch = True
+
+for entry in [entry for entry in importlib_resources.contents('examples') if
+              not pkg_resources.resource_isdir('examples', entry)
+              and entry.endswith(".py")
+              and '__init__' not in entry
+              and 'example_basic_elements_test' not in entry
+              and 'demo.py' not in entry
+              ]:
+    with path('examples', entry) as file_in:
+        print("")
+        print("")
+        print("")
+        print("*****************************************************************")
+        print("Running {}".format(entry))
+        print("*****************************************************************")
+        with swap_attr(sys, "stdin", StringIO("q")):
+            runpy.run_path(file_in, run_name="__main__")

flatland/utils/graphics_pil.py

@@ -33,6 +33,9 @@ from flatland.core.grid.rail_env_grid import RailEnvTransitions  # noqa: E402
 
 
 class PILGL(GraphicsLayer):
+    # hack: in continuous integration, we run multiple
+    unattended_switch = False
+
     def __init__(self, width, height, jupyter=False):
         self.yxBase = (0, 0)
         self.linewidth = 4
@@ -157,9 +160,14 @@ class PILGL(GraphicsLayer):
 
     def open_window(self):
         assert self.window_open is False, "Window is already open!"
-        # use tk.Toplevel() instead of tk.Tk()
-        # https://stackoverflow.com/questions/26097811/image-pyimage2-doesnt-exist
-        self.window_root = tk.Tk()
+        if self.unattended_switch:
+            # use tk.Toplevel() instead of tk.Tk() since we run all examples from the same python script
+            # https://stackoverflow.com/questions/26097811/image-pyimage2-doesnt-exist
+            tk.Toplevel()
+        else:
+            tk.Tk()
+
+        self.window_root = tk.Toplevel()
         self.window_root.withdraw()
         self.window = tk.Toplevel(self.window_root)
         self.window.title("Flatland")

tox.ini

@@ -52,6 +52,7 @@ setenv =
     PYTHONPATH = {toxinidir}
 passenv =
     DISPLAY
+    XAUTHORITY
 ; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
     HTTP_PROXY
     HTTPS_PROXY
@@ -68,6 +69,7 @@ setenv =
     PYTHONPATH = {toxinidir}
 passenv =
     DISPLAY
+    XAUTHORITY
 ; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
     HTTP_PROXY
     HTTPS_PROXY
@@ -94,8 +96,7 @@ deps =
 commands =
 ; run examples from subfolder to ensure that resources are accessed via resources and not via relative paths
     sh -c 'mkdir -p {envtmpdir}/c236d3c240d61a0969d4cb59e2180ce5'
-; pipe echo into python since some examples expect input to close the window after the example is run
-    sh -c 'cd {envtmpdir}/c236d3c240d61a0969d4cb59e2180ce5 && ls {toxinidir}/examples/*.py | xargs -I{} -n 1 sh -c "echo -e \"\n====== Running {} ========\n\"; echo "q" | python {}"'
+    sh -c 'cd {envtmpdir}/c236d3c240d61a0969d4cb59e2180ce5 &&  python {toxinidir}/benchmarks/run_all_examples.py'
 
 [testenv:notebooks]
 basepython = python