Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 21 additions & 26 deletions dpbench/benchmarks/black_scholes/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,36 +2,31 @@
#
# SPDX-License-Identifier: Apache 2.0

from .black_scholes_dpnp import black_scholes as black_scholes_dpnp
from .black_scholes_initialize import initialize
from .black_scholes_numba_dpex_k import (
black_scholes as black_scholes_numba_dpex_k,
)
from .black_scholes_numba_dpex_n import (
black_scholes as black_scholes_numba_dpex_n,
)
from .black_scholes_numba_dpex_p import (
black_scholes as black_scholes_numba_dpex_p,
)
from .black_scholes_numba_n import black_scholes as black_scholes_numba_n
from .black_scholes_numba_np import black_scholes as black_scholes_numba_np
from .black_scholes_numba_npr import black_scholes as black_scholes_numba_npr
from .black_scholes_numpy import black_scholes as black_scholes_numpy
from .black_scholes_python import black_scholes as black_scholes_python
from .black_scholes_sycl_native_ext import black_scholes_sycl

# List of all available benchmarks, each element must either a postfix string
# (in this case specific benchmark module name will be deduced from postfix and
# benchmark name) or 2-element tuple where first element is potfix and second is
# module name.
all_benchmarks = [
"dpnp",
"numba_dpex_k",
"numba_dpex_p",
"numba_dpex_n",
"numba_n",
"numba_np",
"numba_npr",
"numpy",
"python",
(
"sycl",
"black_scholes_sycl_native_ext.black_scholes_sycl._black_scholes_sycl",
),
]

__all__ = [
"initialize",
"black_scholes_dpnp",
"black_scholes_numba_dpex_k",
"black_scholes_numba_dpex_n",
"black_scholes_numba_dpex_p",
"black_scholes_numba_n",
"black_scholes_numba_np",
"black_scholes_numba_npr",
"black_scholes_numpy",
"black_scholes_python",
"black_scholes_sycl",
"all_benchmarks",
]

"""
Expand Down
99 changes: 89 additions & 10 deletions dpbench/infrastructure/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#
# SPDX-License-Identifier: BSD-3-Clause

import importlib
import json
import logging
import os
Expand Down Expand Up @@ -468,7 +469,7 @@ def __init__(self, bench, impl_postfix, preset, repeat, timeout):
with Manager() as manager:
results_dict = manager.dict()
p = Process(
target=tout.exit_after(timeout)(_exec),
target=_exec,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please revert this and update in #166 if needed

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without it it fails on my win machine in 100% cases so I can't test it.

args=(
self.bench,
self.fmwrk,
Expand All @@ -486,7 +487,7 @@ def __init__(self, bench, impl_postfix, preset, repeat, timeout):
),
)
p.start()
res = p.join(timeout * 1.2)
res = p.join(timeout)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please revert this

if res is None and p.exitcode is None:
logging.error(
"Terminating process due to timeout in the execution "
Expand Down Expand Up @@ -520,7 +521,10 @@ def __init__(self, bench, impl_postfix, preset, repeat, timeout):
self.results.results.update(
{outarr: npzfile[outarr]}
)
os.remove(output_npz)
try:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please revert this.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without it it fails on my win machine in 100% cases so I can't test it.

os.remove(output_npz)
except:
pass
if results_dict["return-value"]:
self.results.results.update(
{"return-value": results_dict["return-value"]}
Expand Down Expand Up @@ -624,13 +628,19 @@ def _set_reference_implementation(self, impl_fnlist):
impl_fnlist : The list of implementation function for the
benchmark.
"""

ref_names = ["python", "numpy"]

impl_fnlist = self.impl_fnlist
for name in ref_names:
impl_fnlist = self._get_updated_fnlist(name, impl_fnlist)

ref_impl_fn = None

for fn in impl_fnlist:
if "python" in fn[0]:
ref_impl_fn = fn
elif "numpy" in fn[0]:
ref_impl_fn = fn
for name in ref_names:
if name in fn[0]:
ref_impl_fn = fn

return ref_impl_fn

Expand Down Expand Up @@ -743,9 +753,17 @@ def __init__(self, bmodule: object, bconfig_path: str = None):
benchmark. If none is provided, the default config inside the
package's bench_info directory is used.
"""

self.bname = bmodule.__name__.split(".")[-1]
self.bdata = dict()
self.refdata = dict()

bench_mod = importlib.import_module("dpbench.benchmarks." + self.bname)
if hasattr(bench_mod, "all_benchmarks"):
self.bench_list = bench_mod.all_benchmarks
else:
self.bench_list = None

try:
self._load_benchmark_info(bconfig_path)
self._set_data_initialization_fn(bmodule)
Expand Down Expand Up @@ -786,13 +804,62 @@ def has_impl(self, impl_postfix: str):
else:
return False

def _get_impl_names(self):
if self.bench_list is None:
return None

result = {}
for bench in self.bench_list:
if isinstance(bench, tuple):
postfix = bench[0]
name = bench[1]
else:
postfix = bench
name = self.bname + "_" + postfix

result[postfix] = name

return result

def _get_updated_fnlist(self, impl_postfix: str, impl_fnlist):
names = self._get_impl_names()
canonical_name = self.bname + "_" + impl_postfix
if names:
name = names[impl_postfix]
else:
name = canonical_name

for n, _ in impl_fnlist:
if n == name:
return impl_fnlist

try:
mod = importlib.import_module(
"dpbench.benchmarks." + self.bname + "." + name
)
except ImportError:
logging.exception("Cannot import " + name)
return impl_fnlist

func_names = [self.bname, self.bname + "_" + impl_postfix]
for func in func_names:
if hasattr(mod, func):
return impl_fnlist + [(canonical_name, getattr(mod, func))]

logging.error(
"Unable to find benchmark function for " + impl_postfix + " postfix"
)
return impl_fnlist

def get_impl(self, impl_postfix: str):
if not impl_postfix:
return None

impl_fnlist = self._get_updated_fnlist(impl_postfix, self.impl_fnlist)

fn = [
impl[1]
for impl in self.impl_fnlist
for impl in impl_fnlist
if self.bname + "_" + impl_postfix == impl[0]
]
if len(fn) > 1:
Expand All @@ -812,7 +879,12 @@ def get_impl(self, impl_postfix: str):

def get_framework(self, impl_postfix: str) -> Framework:
try:
return self.impl_to_fw_map[self.bname + "_" + impl_postfix]
impl_fnlist = self._get_updated_fnlist(
impl_postfix, self.impl_fnlist
)
impl_to_fw_map = self._set_impl_to_framework_map(impl_fnlist)

return impl_to_fw_map[self.bname + "_" + impl_postfix]
except KeyError:
logging.exception(
"No framework found for the implementation "
Expand Down Expand Up @@ -905,8 +977,15 @@ def run(
results.append(result)

else:
impl_fnlist = self.get_impl_fnlist()

bench_list = self._get_impl_names()
if bench_list:
for postfix in bench_list.keys():
impl_fnlist = self._get_updated_fnlist(postfix, impl_fnlist)

# Run the benchmark for all available implementations
for impl in self.get_impl_fnlist():
for impl in impl_fnlist:
impl_postfix = impl[0][
(len(self.bname) - len(impl[0]) + 1) : # noqa: E203
]
Expand Down
1 change: 0 additions & 1 deletion dpbench/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ def _print_results(result: dpbi.BenchmarkResults):
+ "implementation:",
result.benchmark_impl_postfix,
)

if result.error_state == ErrorCodes.SUCCESS:
print("framework:", result.framework_name)
print("framework version:", result.framework_version)
Expand Down