diff --git a/dpbench/benchmarks/black_scholes/__init__.py b/dpbench/benchmarks/black_scholes/__init__.py index 44494b95..34898655 100644 --- a/dpbench/benchmarks/black_scholes/__init__.py +++ b/dpbench/benchmarks/black_scholes/__init__.py @@ -2,36 +2,31 @@ # # SPDX-License-Identifier: Apache 2.0 -from .black_scholes_dpnp import black_scholes as black_scholes_dpnp from .black_scholes_initialize import initialize -from .black_scholes_numba_dpex_k import ( - black_scholes as black_scholes_numba_dpex_k, -) -from .black_scholes_numba_dpex_n import ( - black_scholes as black_scholes_numba_dpex_n, -) -from .black_scholes_numba_dpex_p import ( - black_scholes as black_scholes_numba_dpex_p, -) -from .black_scholes_numba_n import black_scholes as black_scholes_numba_n -from .black_scholes_numba_np import black_scholes as black_scholes_numba_np -from .black_scholes_numba_npr import black_scholes as black_scholes_numba_npr -from .black_scholes_numpy import black_scholes as black_scholes_numpy -from .black_scholes_python import black_scholes as black_scholes_python -from .black_scholes_sycl_native_ext import black_scholes_sycl + +# List of all available benchmarks, each element must either a postfix string +# (in this case specific benchmark module name will be deduced from postfix and +# benchmark name) or 2-element tuple where first element is potfix and second is +# module name. +all_benchmarks = [ + "dpnp", + "numba_dpex_k", + "numba_dpex_p", + "numba_dpex_n", + "numba_n", + "numba_np", + "numba_npr", + "numpy", + "python", + ( + "sycl", + "black_scholes_sycl_native_ext.black_scholes_sycl._black_scholes_sycl", + ), +] __all__ = [ "initialize", - "black_scholes_dpnp", - "black_scholes_numba_dpex_k", - "black_scholes_numba_dpex_n", - "black_scholes_numba_dpex_p", - "black_scholes_numba_n", - "black_scholes_numba_np", - "black_scholes_numba_npr", - "black_scholes_numpy", - "black_scholes_python", - "black_scholes_sycl", + "all_benchmarks", ] """ diff --git a/dpbench/infrastructure/benchmark.py b/dpbench/infrastructure/benchmark.py index 15b815ee..5aff4228 100644 --- a/dpbench/infrastructure/benchmark.py +++ b/dpbench/infrastructure/benchmark.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause +import importlib import json import logging import os @@ -468,7 +469,7 @@ def __init__(self, bench, impl_postfix, preset, repeat, timeout): with Manager() as manager: results_dict = manager.dict() p = Process( - target=tout.exit_after(timeout)(_exec), + target=_exec, args=( self.bench, self.fmwrk, @@ -486,7 +487,7 @@ def __init__(self, bench, impl_postfix, preset, repeat, timeout): ), ) p.start() - res = p.join(timeout * 1.2) + res = p.join(timeout) if res is None and p.exitcode is None: logging.error( "Terminating process due to timeout in the execution " @@ -520,7 +521,10 @@ def __init__(self, bench, impl_postfix, preset, repeat, timeout): self.results.results.update( {outarr: npzfile[outarr]} ) - os.remove(output_npz) + try: + os.remove(output_npz) + except: + pass if results_dict["return-value"]: self.results.results.update( {"return-value": results_dict["return-value"]} @@ -624,13 +628,19 @@ def _set_reference_implementation(self, impl_fnlist): impl_fnlist : The list of implementation function for the benchmark. """ + + ref_names = ["python", "numpy"] + + impl_fnlist = self.impl_fnlist + for name in ref_names: + impl_fnlist = self._get_updated_fnlist(name, impl_fnlist) + ref_impl_fn = None for fn in impl_fnlist: - if "python" in fn[0]: - ref_impl_fn = fn - elif "numpy" in fn[0]: - ref_impl_fn = fn + for name in ref_names: + if name in fn[0]: + ref_impl_fn = fn return ref_impl_fn @@ -743,9 +753,17 @@ def __init__(self, bmodule: object, bconfig_path: str = None): benchmark. If none is provided, the default config inside the package's bench_info directory is used. """ + self.bname = bmodule.__name__.split(".")[-1] self.bdata = dict() self.refdata = dict() + + bench_mod = importlib.import_module("dpbench.benchmarks." + self.bname) + if hasattr(bench_mod, "all_benchmarks"): + self.bench_list = bench_mod.all_benchmarks + else: + self.bench_list = None + try: self._load_benchmark_info(bconfig_path) self._set_data_initialization_fn(bmodule) @@ -786,13 +804,62 @@ def has_impl(self, impl_postfix: str): else: return False + def _get_impl_names(self): + if self.bench_list is None: + return None + + result = {} + for bench in self.bench_list: + if isinstance(bench, tuple): + postfix = bench[0] + name = bench[1] + else: + postfix = bench + name = self.bname + "_" + postfix + + result[postfix] = name + + return result + + def _get_updated_fnlist(self, impl_postfix: str, impl_fnlist): + names = self._get_impl_names() + canonical_name = self.bname + "_" + impl_postfix + if names: + name = names[impl_postfix] + else: + name = canonical_name + + for n, _ in impl_fnlist: + if n == name: + return impl_fnlist + + try: + mod = importlib.import_module( + "dpbench.benchmarks." + self.bname + "." + name + ) + except ImportError: + logging.exception("Cannot import " + name) + return impl_fnlist + + func_names = [self.bname, self.bname + "_" + impl_postfix] + for func in func_names: + if hasattr(mod, func): + return impl_fnlist + [(canonical_name, getattr(mod, func))] + + logging.error( + "Unable to find benchmark function for " + impl_postfix + " postfix" + ) + return impl_fnlist + def get_impl(self, impl_postfix: str): if not impl_postfix: return None + impl_fnlist = self._get_updated_fnlist(impl_postfix, self.impl_fnlist) + fn = [ impl[1] - for impl in self.impl_fnlist + for impl in impl_fnlist if self.bname + "_" + impl_postfix == impl[0] ] if len(fn) > 1: @@ -812,7 +879,12 @@ def get_impl(self, impl_postfix: str): def get_framework(self, impl_postfix: str) -> Framework: try: - return self.impl_to_fw_map[self.bname + "_" + impl_postfix] + impl_fnlist = self._get_updated_fnlist( + impl_postfix, self.impl_fnlist + ) + impl_to_fw_map = self._set_impl_to_framework_map(impl_fnlist) + + return impl_to_fw_map[self.bname + "_" + impl_postfix] except KeyError: logging.exception( "No framework found for the implementation " @@ -905,8 +977,15 @@ def run( results.append(result) else: + impl_fnlist = self.get_impl_fnlist() + + bench_list = self._get_impl_names() + if bench_list: + for postfix in bench_list.keys(): + impl_fnlist = self._get_updated_fnlist(postfix, impl_fnlist) + # Run the benchmark for all available implementations - for impl in self.get_impl_fnlist(): + for impl in impl_fnlist: impl_postfix = impl[0][ (len(self.bname) - len(impl[0]) + 1) : # noqa: E203 ] diff --git a/dpbench/runner.py b/dpbench/runner.py index 9a25e98b..f4c99368 100644 --- a/dpbench/runner.py +++ b/dpbench/runner.py @@ -22,7 +22,6 @@ def _print_results(result: dpbi.BenchmarkResults): + "implementation:", result.benchmark_impl_postfix, ) - if result.error_state == ErrorCodes.SUCCESS: print("framework:", result.framework_name) print("framework version:", result.framework_version)