script seems done! takes the average of N times (currently 5) for each benchmark and formats it sort-of well in the output
diff --git a/scripts/bench/bench.py b/scripts/bench/bench.py
index 9118965..370a3b6 100755
--- a/scripts/bench/bench.py
+++ b/scripts/bench/bench.py
@@ -14,10 +14,7 @@
# ==============================================================================
def addCommonFlags(parser):
- compilers = parser.add_mutually_exclusive_group()
- compilers.add_argument("--msvc", action = "store_true", help = "use msvc")
- compilers.add_argument("--gcc", action = "store_true", help = "use gcc")
- compilers.add_argument("--clang", action = "store_true", help = "use clang")
+ parser.add_argument("compiler", choices=['msvc', 'gcc', 'clang'], default='msvc', help = "compiler to use")
parser.add_argument("--debug", action = "store_true", help = "build in debug")
parser.add_argument("--catch", action = "store_true", help = "use Catch instead of doctest")
parser.add_argument("--disabled", action = "store_true", help = "<doctest> define DOCTEST_CONFIG_DISABLE")
@@ -27,7 +24,6 @@
parser.add_argument("--checks", type=int, default=1, help = "number of asserts per test case")
parser.add_argument("--asserts", choices=['normal', 'binary', 'fast'], default="normal",
help = "<doctest> type of assert used - Catch: only normal")
- parser.add_argument("--to-file", action = "store_true", help = "dumps the result to a file named result.txt")
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
@@ -38,7 +34,7 @@
parser_r = subparsers.add_parser('runtime', help='benchmark runtime')
addCommonFlags(parser_r)
parser_r.add_argument("--loop-iters", type=int, default=1000, help = "loop N times all asserts in each test case")
-parser_r.add_argument("--info", action = "store_true", help = "log the variables with INFO()")
+parser_r.add_argument("--info", action = "store_true", help = "log the loop variable with INFO()")
def compile(args): args.compile = True; args.runtime = False
def runtime(args): args.compile = False; args.runtime = True
@@ -170,14 +166,14 @@
# ==============================================================================
compiler = ""
-if args.clang:
+if args.compiler == 'clang':
compiler = " -DCMAKE_CXX_COMPILER=clang++"
-if args.gcc:
+if args.compiler == 'gcc':
compiler = " -DCMAKE_CXX_COMPILER=g++"
# setup cmake command
cmake_command = 'cmake . -G "Visual Studio 15 Win64"' # MSVC 2017
-if not args.msvc:
+if args.compiler != 'msvc':
cmake_command = 'cmake . -G "MinGW Makefiles" -DCMAKE_BUILD_TYPE=' + ('Debug' if args.debug else 'Release')
if os.name != "nt":
cmake_command = 'cmake . -DCMAKE_BUILD_TYPE=' + ('Debug' if args.debug else 'Release')
@@ -189,7 +185,7 @@
# ==============================================================================
the_config = ''
-if args.msvc:
+if args.compiler == 'msvc':
if args.debug: the_config = ' --config Debug'
else: the_config = ' --config Release'
@@ -206,7 +202,7 @@
if args.runtime:
start = datetime.now()
- if args.msvc:
+ if args.compiler == 'msvc':
os.system(('Debug' if args.debug else 'Release') + '\\bench.exe')
elif os.name == "nt":
os.system('bench.exe')
diff --git a/scripts/bench/run_all.py b/scripts/bench/run_all.py
index bd31784..7e220d9 100755
--- a/scripts/bench/run_all.py
+++ b/scripts/bench/run_all.py
@@ -5,6 +5,8 @@
import json
import subprocess
+average_num_times = 5
+
with open('tests.json') as data_file:
data = json.load(data_file)
@@ -16,8 +18,6 @@
return str(line.rsplit(' ', 1)[-1])
return ""
-#print(runBench("python bench.py compile --msvc --debug --files 200 --tests 0"))
-
call = 'python ./bench.py'
the_os = 'linux'
if os.name == "nt":
@@ -25,19 +25,27 @@
the_os = 'windows'
f = open('results.txt', 'w')
-for test in ['header', 'asserts']:
- print('************** ' + test)
+for test in ['header', 'asserts', 'runtime']:
+ print( '\n************** ' + test + '\n')
+ f.write('\n************** ' + test + '\n')
for framework in ['doctest', 'catch']:
- print('== ' + framework)
+ print( '== ' + framework)
+ f.write('== ' + framework)
for config in data['compilers'][the_os]:
- #print(config)
- for curr in data[test]:
+ for curr in data[test][1]:
if curr[0] == framework or curr[0] == "any":
- command = call + config + curr[1] + (' --catch' if framework == 'catch' else '')
+ command = call + data[test][0] + config + curr[1] + (' --catch' if framework == 'catch' else '')
print(command)
- res = runBench(command)
- print(res)
- f.write(res + " ")
+
+ accum = float(0)
+ for i in range(0, average_num_times):
+ res = float(runBench(command))
+ print(res)
+ accum += res
+
+ average = "{:7.2f}".format(round(accum / average_num_times, 2))
+ print("AVERAGE: " + average)
+ f.write(average + " | ")
f.write("\n")
f.close()
diff --git a/scripts/bench/tests.json b/scripts/bench/tests.json
index 677b738..f880dea 100644
--- a/scripts/bench/tests.json
+++ b/scripts/bench/tests.json
@@ -1,31 +1,44 @@
{
"compilers": {
"windows": [
- " compile --msvc --debug",
- " compile --msvc",
- " compile --gcc --debug",
- " compile --gcc"
+ " msvc --debug",
+ " msvc",
+ " gcc --debug",
+ " gcc"
],
"linux": [
- " compile --gcc --debug",
- " compile --gcc",
- " compile --clang --debug",
- " compile --clang"
+ " gcc --debug",
+ " gcc",
+ " clang --debug",
+ " clang"
]
},
"header": [
- ["any", " --files 200 --tests 0"],
- ["any", " --files 200 --tests 0 --implement"],
- ["any", " --files 200 --tests 0 --implement --header"],
- ["doctest", " --files 200 --tests 0 --implement --header --disabled"]
+ " compile",
+ [
+ ["any", " --files 200 --tests 0"],
+ ["any", " --files 200 --tests 0 --implement"],
+ ["any", " --files 200 --tests 0 --implement --header"],
+ ["doctest", " --files 200 --tests 0 --implement --header --disabled"]
+ ]
],
"asserts": [
- ["any", " --header --files 10 --tests 0 --checks 0"],
- ["any", " --header --files 10 --tests 50 --checks 100 --asserts normal"],
- ["doctest", " --header --files 10 --tests 50 --checks 100 --asserts binary"],
- ["doctest", " --header --files 10 --tests 50 --checks 100 --asserts fast"],
- ["doctest", " --header --files 10 --tests 50 --checks 100 --asserts fast --fast"],
- ["doctest", " --header --files 10 --tests 50 --checks 100 --asserts normal --disabled"],
- ["catch", " --header --files 10 --tests 50 --checks 100 --asserts normal --fast"]
+ " compile",
+ [
+ ["any", " --header --files 10 --tests 0 --checks 0"],
+ ["any", " --header --files 10 --tests 50 --checks 100 --asserts normal"],
+ ["doctest", " --header --files 10 --tests 50 --checks 100 --asserts binary"],
+ ["doctest", " --header --files 10 --tests 50 --checks 100 --asserts fast"],
+ ["doctest", " --header --files 10 --tests 50 --checks 100 --asserts fast --fast"],
+ ["doctest", " --header --files 10 --tests 50 --checks 100 --asserts normal --disabled"],
+ ["catch", " --header --files 10 --tests 50 --checks 100 --asserts normal --fast"]
+ ]
+ ],
+ "runtime": [
+ " runtime",
+ [
+ ["any", " --header --files 1 --tests 1 --checks 1 --loop-iters 10000000"],
+ ["any", " --header --files 1 --tests 1 --checks 1 --loop-iters 10000000 --info"]
+ ]
]
}