#65 benchmark and profile all examples in scheduled pipeline instead of on every commit

.gitlab-ci.yml

@@ -62,6 +62,6 @@ benchmarks_and_profiling:
         - apt update
         - apt install -y libgl1-mesa-glx xvfb graphviz xdg-utils libcairo2-dev libjpeg-dev libgif-dev
         - pip install tox
-        - xvfb-run tox -e benchmarks -v --recreate
+        - xvfb-run tox -e benchmarks,profiling -v --recreate
 
 

benchmarks/benchmark_all_examples.py

+import runpy
+import sys
+from io import StringIO
+from test.support import swap_attr
+from time import sleep
+
+import importlib_resources
+import pkg_resources
+from benchmarker import Benchmarker
+from importlib_resources import path
+
+for entry in [entry for entry in importlib_resources.contents('examples') if
+              not pkg_resources.resource_isdir('examples', entry)
+              and entry.endswith(".py")
+              and '__init__' not in entry
+              and 'demo.py' not in entry
+              ]:
+    print("*****************************************************************")
+    print("Benchmarking {}".format(entry))
+    print("*****************************************************************")
+
+    with path('examples', entry) as file_in:
+        with Benchmarker(cycle=20, extra=1) as bench:
+            @bench(entry)
+            def _(_):
+                # prevent Benchmarker from doing "ZeroDivisionError: float division by zero:
+                #    ratio = base_time / real_time"
+                sleep(0.001)
+                # In order to pipe input into examples that have input(),
+                # we use the test package, which is meant for internal use by Python only internal and
+                # Any use of this package outside of Python’s standard library is discouraged as code (..)
+                # can change or be removed without notice between releases of Python.
+                # https://docs.python.org/3/library/test.html
+                # TODO remove input() from examples?
+                with swap_attr(sys, "stdin", StringIO("q")):
+                    runpy.run_path(file_in, run_name="__main__")

benchmarks/profile_all_examples.py

+import cProfile
+import runpy
+import sys
+from io import StringIO
+from test.support import swap_attr
+
+import importlib_resources
+import pkg_resources
+from importlib_resources import path
+
+
+def profile(resource, entry):
+    with path(resource, entry) as file_in:
+        # we use the test package, which is meant for internal use by Python only internal and
+        # Any use of this package outside of Python’s standard library is discouraged as code (..)
+        # can change or be removed without notice between releases of Python.
+        # https://docs.python.org/3/library/test.html
+        # TODO remove input() from examples
+        print("*****************************************************************")
+        print("Profiling {}".format(entry))
+        print("*****************************************************************")
+        with swap_attr(sys, "stdin", StringIO("q")):
+            global my_func
+
+            def my_func(): runpy.run_path(file_in, run_name="__main__")
+
+            cProfile.run('my_func()', sort='time')
+
+
+for entry in [entry for entry in importlib_resources.contents('examples') if
+              not pkg_resources.resource_isdir('examples', entry)
+              and entry.endswith(".py")
+              and '__init__' not in entry
+              and 'demo.py' not in entry
+              ]:
+    profile('examples', entry)

benchmarks/complex_rail_benchmark.py → examples/complex_rail_benchmark.py

@@ -2,7 +2,6 @@
 import random
 
 import numpy as np
-from benchmarker import Benchmarker
 
 from flatland.envs.generators import complex_rail_generator
 from flatland.envs.rail_env import RailEnv
@@ -65,7 +64,4 @@ def run_benchmark():
 
 
 if __name__ == "__main__":
-    with Benchmarker(cycle=20, extra=1) as bench:
-        @bench("Everything")
-        def _(bm):
-            run_benchmark()
+    run_benchmark()

examples/demo.py

@@ -150,12 +150,14 @@ class Demo:
     def run_example_flatland_000():
         demo_flatland_000 = Demo(Scenario_Generator.load_scenario('example_flatland_000.pkl'))
         demo_flatland_000.renderer.resize()
+        demo_flatland_000.set_max_framerate(5)
         demo_flatland_000.run_demo(60)
 
     @staticmethod
     def run_example_flatland_001():
         demo_flatland_000 = Demo(Scenario_Generator.load_scenario('example_flatland_001.pkl'))
         demo_flatland_000.renderer.resize()
+        demo_flatland_000.set_max_framerate(5)
         demo_flatland_000.set_record_frames(os.path.join(__file_dirname__, '..', 'rendering', 'frame_{:04d}.bmp'))
         demo_flatland_000.run_demo(60)
 

flatland/utils/graphics_pil.py

@@ -129,7 +129,9 @@ class PILGL(GraphicsLayer):
 
     def open_window(self):
         assert self.window_open is False, "Window is already open!"
-        self.window = tk.Tk()
+        # use tk.Toplevel() instead of tk.Tk()
+        # https://stackoverflow.com/questions/26097811/image-pyimage2-doesnt-exist
+        self.window = tk.Toplevel()
         self.window.title("Flatland")
         self.window.configure(background='grey')
         self.window_open = True

tox.ini

@@ -60,7 +60,23 @@ deps =
     -r{toxinidir}/requirements_dev.txt
     -r{toxinidir}/requirements_continuous_integration.txt
 commands =
-    sh -c 'ls benchmarks/*.py  | xargs -n 1 python'
+    python benchmarks/benchmark_all_examples.py
+
+[testenv:profiling]
+basepython = python
+setenv =
+    PYTHONPATH = {toxinidir}
+passenv =
+    DISPLAY
+; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
+    HTTP_PROXY
+    HTTPS_PROXY
+whitelist_externals = sh
+deps =
+    -r{toxinidir}/requirements_dev.txt
+    -r{toxinidir}/requirements_continuous_integration.txt
+commands =
+    python benchmarks/profile_all_examples.py
 
 [testenv:examples]
 basepython = python