From 244be2d93bab6162f5df910d1c3a54f0b266e5f5 Mon Sep 17 00:00:00 2001
From: u214892 <u214892@sbb.ch>
Date: Tue, 18 Jun 2019 11:16:23 +0200
Subject: [PATCH] #65 benchmark and profile all examples in scheduled pipeline
instead of on every commit
---
.gitlab-ci.yml | 2 +-
benchmarks/benchmark_all_examples.py | 36 +++++++++++++++++++
benchmarks/profile_all_examples.py | 36 +++++++++++++++++++
.../complex_rail_benchmark.py | 6 +---
examples/demo.py | 2 ++
flatland/utils/graphics_pil.py | 4 ++-
tox.ini | 18 +++++++++-
7 files changed, 96 insertions(+), 8 deletions(-)
create mode 100644 benchmarks/benchmark_all_examples.py
create mode 100644 benchmarks/profile_all_examples.py
rename {benchmarks => examples}/complex_rail_benchmark.py (92%)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9ea7d59e..c2002409 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -62,6 +62,6 @@ benchmarks_and_profiling:
- apt update
- apt install -y libgl1-mesa-glx xvfb graphviz xdg-utils libcairo2-dev libjpeg-dev libgif-dev
- pip install tox
- - xvfb-run tox -e benchmarks -v --recreate
+ - xvfb-run tox -e benchmarks,profiling -v --recreate
diff --git a/benchmarks/benchmark_all_examples.py b/benchmarks/benchmark_all_examples.py
new file mode 100644
index 00000000..676bfe16
--- /dev/null
+++ b/benchmarks/benchmark_all_examples.py
@@ -0,0 +1,36 @@
+import runpy
+import sys
+from io import StringIO
+from test.support import swap_attr
+from time import sleep
+
+import importlib_resources
+import pkg_resources
+from benchmarker import Benchmarker
+from importlib_resources import path
+
+for entry in [entry for entry in importlib_resources.contents('examples') if
+ not pkg_resources.resource_isdir('examples', entry)
+ and entry.endswith(".py")
+ and '__init__' not in entry
+ and 'demo.py' not in entry
+ ]:
+ print("*****************************************************************")
+ print("Benchmarking {}".format(entry))
+ print("*****************************************************************")
+
+ with path('examples', entry) as file_in:
+ with Benchmarker(cycle=20, extra=1) as bench:
+ @bench(entry)
+ def _(_):
+ # prevent Benchmarker from doing "ZeroDivisionError: float division by zero:
+ # ratio = base_time / real_time"
+ sleep(0.001)
+ # In order to pipe input into examples that have input(),
+ # we use the test package, which is meant for internal use by Python only internal and
+ # Any use of this package outside of Python’s standard library is discouraged as code (..)
+ # can change or be removed without notice between releases of Python.
+ # https://docs.python.org/3/library/test.html
+ # TODO remove input() from examples?
+ with swap_attr(sys, "stdin", StringIO("q")):
+ runpy.run_path(file_in, run_name="__main__")
diff --git a/benchmarks/profile_all_examples.py b/benchmarks/profile_all_examples.py
new file mode 100644
index 00000000..0b6db571
--- /dev/null
+++ b/benchmarks/profile_all_examples.py
@@ -0,0 +1,36 @@
+import cProfile
+import runpy
+import sys
+from io import StringIO
+from test.support import swap_attr
+
+import importlib_resources
+import pkg_resources
+from importlib_resources import path
+
+
+def profile(resource, entry):
+ with path(resource, entry) as file_in:
+ # we use the test package, which is meant for internal use by Python only internal and
+ # Any use of this package outside of Python’s standard library is discouraged as code (..)
+ # can change or be removed without notice between releases of Python.
+ # https://docs.python.org/3/library/test.html
+ # TODO remove input() from examples
+ print("*****************************************************************")
+ print("Profiling {}".format(entry))
+ print("*****************************************************************")
+ with swap_attr(sys, "stdin", StringIO("q")):
+ global my_func
+
+ def my_func(): runpy.run_path(file_in, run_name="__main__")
+
+ cProfile.run('my_func()', sort='time')
+
+
+for entry in [entry for entry in importlib_resources.contents('examples') if
+ not pkg_resources.resource_isdir('examples', entry)
+ and entry.endswith(".py")
+ and '__init__' not in entry
+ and 'demo.py' not in entry
+ ]:
+ profile('examples', entry)
diff --git a/benchmarks/complex_rail_benchmark.py b/examples/complex_rail_benchmark.py
similarity index 92%
rename from benchmarks/complex_rail_benchmark.py
rename to examples/complex_rail_benchmark.py
index 1f6985dc..44e4b534 100644
--- a/benchmarks/complex_rail_benchmark.py
+++ b/examples/complex_rail_benchmark.py
@@ -2,7 +2,6 @@
import random
import numpy as np
-from benchmarker import Benchmarker
from flatland.envs.generators import complex_rail_generator
from flatland.envs.rail_env import RailEnv
@@ -65,7 +64,4 @@ def run_benchmark():
if __name__ == "__main__":
- with Benchmarker(cycle=20, extra=1) as bench:
- @bench("Everything")
- def _(bm):
- run_benchmark()
+ run_benchmark()
diff --git a/examples/demo.py b/examples/demo.py
index 40ebeb4c..0def233d 100644
--- a/examples/demo.py
+++ b/examples/demo.py
@@ -150,12 +150,14 @@ class Demo:
def run_example_flatland_000():
demo_flatland_000 = Demo(Scenario_Generator.load_scenario('example_flatland_000.pkl'))
demo_flatland_000.renderer.resize()
+ demo_flatland_000.set_max_framerate(5)
demo_flatland_000.run_demo(60)
@staticmethod
def run_example_flatland_001():
demo_flatland_000 = Demo(Scenario_Generator.load_scenario('example_flatland_001.pkl'))
demo_flatland_000.renderer.resize()
+ demo_flatland_000.set_max_framerate(5)
demo_flatland_000.set_record_frames(os.path.join(__file_dirname__, '..', 'rendering', 'frame_{:04d}.bmp'))
demo_flatland_000.run_demo(60)
diff --git a/flatland/utils/graphics_pil.py b/flatland/utils/graphics_pil.py
index 23f8b879..50bc723a 100644
--- a/flatland/utils/graphics_pil.py
+++ b/flatland/utils/graphics_pil.py
@@ -129,7 +129,9 @@ class PILGL(GraphicsLayer):
def open_window(self):
assert self.window_open is False, "Window is already open!"
- self.window = tk.Tk()
+ # use tk.Toplevel() instead of tk.Tk()
+ # https://stackoverflow.com/questions/26097811/image-pyimage2-doesnt-exist
+ self.window = tk.Toplevel()
self.window.title("Flatland")
self.window.configure(background='grey')
self.window_open = True
diff --git a/tox.ini b/tox.ini
index 43a02f37..04cdc112 100644
--- a/tox.ini
+++ b/tox.ini
@@ -60,7 +60,23 @@ deps =
-r{toxinidir}/requirements_dev.txt
-r{toxinidir}/requirements_continuous_integration.txt
commands =
- sh -c 'ls benchmarks/*.py | xargs -n 1 python'
+ python benchmarks/benchmark_all_examples.py
+
+[testenv:profiling]
+basepython = python
+setenv =
+ PYTHONPATH = {toxinidir}
+passenv =
+ DISPLAY
+; HTTP_PROXY+HTTPS_PROXY required behind corporate proxies
+ HTTP_PROXY
+ HTTPS_PROXY
+whitelist_externals = sh
+deps =
+ -r{toxinidir}/requirements_dev.txt
+ -r{toxinidir}/requirements_continuous_integration.txt
+commands =
+ python benchmarks/profile_all_examples.py
[testenv:examples]
basepython = python
--
GitLab