From 810afd7d2b9c7404c94bf92d72542266986a9217 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 14 Oct 2025 13:39:23 -0700 Subject: [PATCH 01/17] update build_locally script to avoid python setup.py develop call --- scripts/build_locally.py | 105 ++++++++++++++++----------------------- 1 file changed, 44 insertions(+), 61 deletions(-) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index ee76204c08..3f4627c2ee 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -33,72 +33,55 @@ def run( target_cuda=None, target_hip=None, ): - build_system = None + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + env = os.environ.copy() - if "linux" in sys.platform: - build_system = "Ninja" - elif sys.platform in ["win32", "cygwin"]: - build_system = "Ninja" - else: - assert False, sys.platform + " not supported" + cmake_args = [] + if c_compiler: + cmake_args.append(f"-DCMAKE_C_COMPILER:PATH={c_compiler}") + if cxx_compiler: + cmake_args.append(f"-DCMAKE_CXX_COMPILER:PATH={cxx_compiler}") + + cmake_args.append(f"-DCMAKE_BUILD_TYPE={build_type}") + cmake_args.append( + f"-DDPCTL_ENABLE_L0_PROGRAM_CREATION={'ON' if level_zero else 'OFF'}" + ) + cmake_args.append(f"-DDPCTL_ENABLE_GLOG={'ON' if use_glog else 'OFF'}") - setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - cmake_args = [ - sys.executable, - "setup.py", - "develop", - ] - if cmake_executable: - cmake_args += [ - "--cmake-executable=" + cmake_executable, - ] - cmake_args += [ - "--build-type=" + build_type, - "--generator=" + build_system, - "--", - "-DCMAKE_C_COMPILER:PATH=" + c_compiler, - "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, - "-DDPCTL_ENABLE_L0_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), - "-DDPCTL_ENABLE_GLOG:BOOL=" + ("ON" if use_glog else "OFF"), - ] if verbose: - cmake_args += [ - "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", - ] - if cmake_opts: - cmake_args += cmake_opts.split() - if target_cuda is not None: + cmake_args.append("-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON") + + if cmake_executable: + cmake_args.append(f"-DCMAKE_EXECUTABLE:PATH={cmake_executable}") + + if target_cuda: if not target_cuda.strip(): - raise ValueError( - "--target-cuda can not be an empty string. " - "Use --target-cuda= or --target-cuda" - ) - if any(opt.startswith("-DDPCTL_TARGET_CUDA=") for opt in cmake_args): - raise ValueError( - "Both --target-cuda and -DDPCTL_TARGET_CUDA in --cmake-opts " - "were specified. Please use only one method " - "to avoid ambiguity" - ) - cmake_args += [ - f"-DDPCTL_TARGET_CUDA={target_cuda}", - ] - if target_hip is not None: + raise ValueError("--target-cuda cannot be empty") + cmake_args.append(f"-DDPCTL_TARGET_CUDA={target_cuda}") + + if target_hip: if not target_hip.strip(): - raise ValueError( - "--target-hip requires an architecture (e.g., gfx90a)" - ) - if any(opt.startswith("-DDPCTL_TARGET_HIP=") for opt in cmake_args): - raise ValueError( - "Both --target-hip and -DDPCTL_TARGET_HIP in --cmake-opts " - "were specified. Please use only one method " - "to avoid ambiguity" - ) - cmake_args += [ - f"-DDPCTL_TARGET_HIP={target_hip}", - ] - subprocess.check_call( - cmake_args, shell=False, cwd=setup_dir, env=os.environ - ) + raise ValueError("--target_hip cannot be empty") + cmake_args.append(f"-DDPCTL_TARGET_HIP={target_hip}") + + env["CMAKE_ARGS"] = " ".join(cmake_args) + + # build the Cmake extensions in-place + build_cmd = [sys.executable, "setup.py", "build_ext", "--inplace"] + subprocess.check_call(build_cmd, cwd=setup_dir, env=env) + + # editable install with pip + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--no-build-isolation", + "--editable", + ".", + ] + + subprocess.check_call(cmd, cwd=setup_dir, env=env) if __name__ == "__main__": From 32c7b9f0ca0e90a4685ceca719a49f5a0e456b63 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 14 Oct 2025 20:02:10 -0700 Subject: [PATCH 02/17] further update build_locally script --- scripts/build_locally.py | 348 +++++++++++++++++++++++---------------- 1 file changed, 205 insertions(+), 143 deletions(-) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 3f4627c2ee..21e02e5684 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -14,160 +14,136 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import os +import shutil import subprocess import sys -def run( - use_oneapi=True, - build_type="Release", - c_compiler=None, - cxx_compiler=None, - level_zero=True, - compiler_root=None, - cmake_executable=None, - use_glog=False, - verbose=False, - cmake_opts="", - target_cuda=None, - target_hip=None, -): - setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - env = os.environ.copy() - - cmake_args = [] - if c_compiler: - cmake_args.append(f"-DCMAKE_C_COMPILER:PATH={c_compiler}") - if cxx_compiler: - cmake_args.append(f"-DCMAKE_CXX_COMPILER:PATH={cxx_compiler}") - - cmake_args.append(f"-DCMAKE_BUILD_TYPE={build_type}") - cmake_args.append( - f"-DDPCTL_ENABLE_L0_PROGRAM_CREATION={'ON' if level_zero else 'OFF'}" - ) - cmake_args.append(f"-DDPCTL_ENABLE_GLOG={'ON' if use_glog else 'OFF'}") - - if verbose: - cmake_args.append("-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON") - - if cmake_executable: - cmake_args.append(f"-DCMAKE_EXECUTABLE:PATH={cmake_executable}") - - if target_cuda: - if not target_cuda.strip(): - raise ValueError("--target-cuda cannot be empty") - cmake_args.append(f"-DDPCTL_TARGET_CUDA={target_cuda}") - - if target_hip: - if not target_hip.strip(): - raise ValueError("--target_hip cannot be empty") - cmake_args.append(f"-DDPCTL_TARGET_HIP={target_hip}") +def run(cmd, env=None, cwd=None): + print("+", " ".join(cmd)) + subprocess.check_call(cmd, env=env, cwd=cwd or os.getcwd()) - env["CMAKE_ARGS"] = " ".join(cmake_args) - # build the Cmake extensions in-place - build_cmd = [sys.executable, "setup.py", "build_ext", "--inplace"] - subprocess.check_call(build_cmd, cwd=setup_dir, env=env) +def _warn(msg: str): + print(f"[build_locally][error] {msg}", file=sys.stderr) - # editable install with pip - cmd = [ - sys.executable, - "-m", - "pip", - "install", - "--no-build-isolation", - "--editable", - ".", - ] - subprocess.check_call(cmd, cwd=setup_dir, env=env) +def _err(msg: str): + print(f"[build_locally][error] {msg}", file=sys.stderr) -if __name__ == "__main__": - import argparse +def parse_args(): + p = argparse.ArgumentParser(description="Local dpctl build driver") - parser = argparse.ArgumentParser( - description="Driver to build dpctl for in-place installation" + p.add_argument( + "--c-compiler", default=None, help="Path or name of C compiler" + ) + p.add_argument( + "--cxx-compiler", default=None, help="Path or name of C++ compiler" ) - driver = parser.add_argument_group(title="Coverage driver arguments") - driver.add_argument("--c-compiler", help="Name of C compiler", default=None) - driver.add_argument( - "--cxx-compiler", help="Name of C++ compiler", default=None + p.add_argument( + "--compiler-root", + type=str, + default=None, + help="Path to compiler installation root", ) - driver.add_argument( + + p.add_argument( "--oneapi", - help="Is one-API installation", dest="oneapi", action="store_true", + help="Use default oneAPI compiler layout", ) - driver.add_argument( + p.add_argument( "--debug", - default="Release", + dest="build_type", const="Debug", action="store_const", - help="Set the compilation mode to debugging", + default="Release", + help="Set build type to Debug (defaults to Release)", ) - driver.add_argument( - "--compiler-root", - type=str, - help="Path to compiler home directory", - default=None, + + p.add_argument( + "--generator", type=str, default="Ninja", help="CMake generator" ) - driver.add_argument( + p.add_argument( "--cmake-executable", type=str, - help="Path to cmake executable", default=None, + help="Path to CMake executable used by build", ) - driver.add_argument( - "--no-level-zero", - help="Enable Level Zero support", - dest="level_zero", - action="store_false", - ) - driver.add_argument( + + p.add_argument( "--glog", - help="DPCTLSyclInterface uses Google logger", dest="glog", action="store_true", + help="Enable DPCTL Google logger support", ) - driver.add_argument( + p.add_argument( "--verbose", - help="Build using vebose makefile mode", dest="verbose", action="store_true", + help="Enable verbose makefile output", + ) + + p.add_argument( + "--no-level-zero", + dest="no_level_zero", + action="store_true", + default=False, + help="Disable Level Zero backend (deprecated: use --target-level-zero " + "OFF)", ) - driver.add_argument( + p.add_argument( + "--target-level-zero", + action="store_true", + help="Enable Level Zero backend explicitly", + ) + + p.add_argument( "--cmake-opts", - help="Options to pass through to cmake", - dest="cmake_opts", - default="", type=str, + default="", + help="Additional options to pass directly to CMake", ) - driver.add_argument( + + p.add_argument( "--target-cuda", nargs="?", const="ON", - help="Enable CUDA target for build; " - "optionally specify architecture (e.g., --target-cuda=sm_80)", default=None, - type=str, + help="Enable CUDA build. Architecture is optional to specify.", ) - driver.add_argument( + p.add_argument( "--target-hip", required=False, - help="Enable HIP target for build. " - "Must specify HIP architecture (e.g., --target-hip=gfx90a)", type=str, + help="Enable HIP backend. Architecture required to be specified.", ) - args = parser.parse_args() - args_to_validate = [ - "c_compiler", - "cxx_compiler", - "compiler_root", - ] + p.add_argument( + "--build-dir", + default="build", + help="CMake build directory (default: build)", + ) + p.add_argument( + "--clean", + action="store_false", + help="Remove build dir before rebuild (default: False)", + ) + p.add_argument( + "--skip-editable", + action="store_true", + help="Skip pip editable install step", + ) + + return p.parse_args() + + +def resolve_compilers(args): + is_linux = "linux" in sys.platform if args.oneapi or ( args.c_compiler is None @@ -175,47 +151,133 @@ def run( and args.compiler_root is None ): args.c_compiler = "icx" - args.cxx_compiler = "icpx" if "linux" in sys.platform else "icx" + args.cxx_compiler = "icpx" if is_linux else "icx" args.compiler_root = None + return + + cr = args.compiler_root + if isinstance(cr, str) and os.path.exists(cr): + if args.c_compiler is None: + args.c_compiler = "icx" + if args.cxx_compiler is None: + args.cxx_compiler = "icpx" if is_linux else "icx" else: - cr = args.compiler_root - if isinstance(cr, str) and os.path.exists(cr): - if args.c_compiler is None: - args.c_compiler = "icx" - if args.cxx_compiler is None: - args.cxx_compiler = "icpx" if "linux" in sys.platform else "icx" - else: + raise RuntimeError( + "'compiler-root' option must be set when using non-default DPC++ " + "layout" + ) + + for opt_name in ("c_compiler", "cxx_compiler"): + arg = getattr(args, opt_name) + if not arg: + continue + if not os.path.exists(arg): + probe = os.path.join(cr, arg) + if os.path.exists(probe): + setattr(args, opt_name, probe) + continue + if not os.path.exists(getattr(args, opt_name)): raise RuntimeError( - "Option 'compiler-root' must be provided when " - "using non-default DPC++ layout." + f"{opt_name.replace('_', '-')} value {arg} not found" ) - args_to_validate = [ - "c_compiler", - "cxx_compiler", - ] - for p in args_to_validate: - arg = getattr(args, p) - assert isinstance(arg, str) - if not os.path.exists(arg): - arg2 = os.path.join(cr, arg) - if os.path.exists(arg2): - arg = arg2 - setattr(args, p, arg) - if not os.path.exists(arg): - opt_name = p.replace("_", "-") - raise RuntimeError(f"Option {opt_name} value {arg} must exist.") + +def main(): + if sys.platform not in ["cygwin", "win32", "linux"]: + _err(f"{sys.platform} not supported") + args = parse_args() + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + build_dir = os.path.join(setup_dir, args.build_dir) + + resolve_compilers(args) + + # clean build dir if --clean set + if args.clean and os.path.exists(build_dir): + print(f"[build_locally] Cleaning build directory: {build_dir}") + shutil.rmtree(build_dir) + + env = os.environ.copy() + + # ignore pre-existing CMAKE_ARGS for determinism in build driver + if "CMAKE_ARGS" in env and env["CMAKE_ARGS"].strip(): + _warn("Ignoring pre-existing CMAKE_ARGS in environment") + del env["CMAKE_ARGS"] + + cmake_defs = [] + + # handle architecture conflicts + if args.target_hip is not None and not args.target_hip.strip(): + _err("--target-hip requires an explicit architecture") + + if args.no_level_zero and args.target_level_zero: + _err("Cannot combine --no-level-zero and --target-level-zero") + + # CUDA/HIP targets + if args.target_cuda: + cmake_defs.append(f"-DDPCTL_TARGET_CUDA={args.target_cuda}") + if args.target_hip: + cmake_defs.append(f"-DDPCTL_TARGET_HIP={args.target_hip}") + + # Level Zero state (on unless explicitly disabled) + if args.no_level_zero: + level_zero_enabled = False + elif args.target_level_zero: + level_zero_enabled = True + else: + level_zero_enabled = True + cmake_defs.append( + "-DDPCTL_ENABLE_L0_PROGRAM_CREATION=" + f"{'ON' if level_zero_enabled else 'OFF'}" + ) + + # compilers and generator + if args.c_compiler: + cmake_defs.append(f"-DCMAKE_C_COMPILER:PATH={args.c_compiler}") + if args.cxx_compiler: + cmake_defs.append(f"-DCMAKE_CXX_COMPILER:PATH={args.cxx_compiler}") + if args.generator: + cmake_defs.append(f"-G{args.generator}") + + cmake_defs.append( + f"-DDPCTL_ENABLE_GLOG:BOOL={'ON' if args.glog else 'OFF'}" + ) + cmake_defs.append(f"-DCMAKE_BUILD_TYPE={args.build_type}") + if args.verbose: + cmake_defs.append("-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON") + + if args.cmake_opts: + cmake_defs.extend(args.cmake_opts.split()) + + env["CMAKE_ARGS"] = " ".join(cmake_defs) + print(f"[build_locally] CMake args:\n {' '.join(cmake_defs)}") + + print("[build_locally] Building extensions in-place...") run( - use_oneapi=args.oneapi, - build_type=args.debug, - c_compiler=args.c_compiler, - cxx_compiler=args.cxx_compiler, - level_zero=args.level_zero, - compiler_root=args.compiler_root, - cmake_executable=args.cmake_executable, - use_glog=args.glog, - verbose=args.verbose, - cmake_opts=args.cmake_opts, - target_cuda=args.target_cuda, - target_hip=args.target_hip, + [sys.executable, "setup.py", "build_ext", "--inplace"], + env=env, + cwd=setup_dir, ) + + if not args.skip_editable: + print("[build_locally] Installing dpctl in editable mode") + run( + [ + sys.executable, + "-m", + "pip", + "install", + "-e", + ".", + "--no-build-isolation", + ], + env=env, + cwd=setup_dir, + ) + else: + print("[build_locally] Skipping editable install (--skip-editable)") + + print("[build_locally] Build complete") + + +if __name__ == "__main__": + main() From 32f150916b84c5d8040bbd0b308372273ac57dc9 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 14 Oct 2025 22:42:04 -0700 Subject: [PATCH 03/17] refactor common build functionality out into separate file to be reused in gen_docs and gen_coverage --- scripts/_build_helper.py | 127 ++++++++++++++++++++++++++++++ scripts/build_locally.py | 166 ++++++++++++--------------------------- 2 files changed, 177 insertions(+), 116 deletions(-) create mode 100644 scripts/_build_helper.py diff --git a/scripts/_build_helper.py b/scripts/_build_helper.py new file mode 100644 index 0000000000..ede078a60a --- /dev/null +++ b/scripts/_build_helper.py @@ -0,0 +1,127 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2025 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import subprocess +import sys + + +def run(cmd, env=None, cwd=None): + print("+", " ".join(cmd)) + subprocess.check_call( + cmd, env=env or os.environ.copy(), cwd=cwd or os.getcwd() + ) + + +def warn(msg: str): + print(f"[build_locally][error] {msg}", file=sys.stderr) + + +def err(msg: str): + print(f"[build_locally][error] {msg}", file=sys.stderr) + + +def resolve_compilers( + oneapi: bool, c_compiler: str, cxx_compiler: str, compiler_root: str +): + is_linux = "linux" in sys.platform + + if oneapi or ( + c_compiler is None and cxx_compiler is None and compiler_root is None + ): + return "icx", ("icpx" if is_linux else "icx"), None + + if not compiler_root or not os.path.exists(compiler_root): + raise RuntimeError( + "--compiler-root option must be set when using non-default DPC++ " + "layout" + ) + + # default values + if c_compiler is None: + c_compiler = "icx" + if cxx_compiler is None: + cxx_compiler = "icpx" if is_linux else "icx" + + for name, opt_name in ( + (c_compiler, "--c-compiler"), + (cxx_compiler, "--cxx-compiler"), + ): + path = ( + name if os.path.exists(name) else os.path.join(compiler_root, name) + ) + if not os.path.exists(path): + raise RuntimeError(f"{opt_name} value {name} not found") + return c_compiler, cxx_compiler, compiler_root + + +def make_cmake_args( + build_type="Release", + c_compiler=None, + cxx_compiler=None, + level_zero=True, + glog=False, + generator=None, + verbose=False, + other_opts="", +): + args = [ + f"-DCMAKE_BUILD_TYPE={build_type}", + f"-DCMAKE_C_COMPILER:PATH={c_compiler}" if c_compiler else "", + f"-DCMAKE_CXX_COMPILER:PATH={cxx_compiler}" if cxx_compiler else "", + f"-DDPCTL_ENABLE_L0_PROGRAM_CREATION={'ON' if level_zero else 'OFF'}", + f"-DDPTL_ENABLE_GLOG:BOOL={'ON' if glog else 'OFF'}", + ] + + if generator: + args.append(f"-G{generator}") + if verbose: + args.append("-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON") + if other_opts: + args.extend(other_opts.split()) + + return " ".join(filter(None, args)) + + +def build_extension(setup_dir, env): + run( + [sys.executable, "setup.py", "build_ext", "--inplace"], + env=env, + cwd=setup_dir, + ) + + +def install_editable(setup_dir, env): + run( + [ + sys.executable, + "-m", + "pip", + "install", + "-e", + ".", + "--no-build-isolation", + ], + env=env, + cwd=setup_dir, + ) + + +def clean_build_dir(build_dir): + if os.path.exists(build_dir): + print(f"Cleaning build directory: {build_dir}") + shutil.rmtree(build_dir) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 21e02e5684..ec1fde44ca 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -16,22 +16,20 @@ import argparse import os -import shutil -import subprocess import sys +# add scripts dir to Python path so we can import _build_helper +sys.path.insert(0, os.path.abspath("scripts")) -def run(cmd, env=None, cwd=None): - print("+", " ".join(cmd)) - subprocess.check_call(cmd, env=env, cwd=cwd or os.getcwd()) - - -def _warn(msg: str): - print(f"[build_locally][error] {msg}", file=sys.stderr) - - -def _err(msg: str): - print(f"[build_locally][error] {msg}", file=sys.stderr) +from _build_helper import ( # noqa: E402 + build_extension, + clean_build_dir, + err, + install_editable, + make_cmake_args, + resolve_compilers, + warn, +) def parse_args(): @@ -142,81 +140,23 @@ def parse_args(): return p.parse_args() -def resolve_compilers(args): - is_linux = "linux" in sys.platform - - if args.oneapi or ( - args.c_compiler is None - and args.cxx_compiler is None - and args.compiler_root is None - ): - args.c_compiler = "icx" - args.cxx_compiler = "icpx" if is_linux else "icx" - args.compiler_root = None - return - - cr = args.compiler_root - if isinstance(cr, str) and os.path.exists(cr): - if args.c_compiler is None: - args.c_compiler = "icx" - if args.cxx_compiler is None: - args.cxx_compiler = "icpx" if is_linux else "icx" - else: - raise RuntimeError( - "'compiler-root' option must be set when using non-default DPC++ " - "layout" - ) - - for opt_name in ("c_compiler", "cxx_compiler"): - arg = getattr(args, opt_name) - if not arg: - continue - if not os.path.exists(arg): - probe = os.path.join(cr, arg) - if os.path.exists(probe): - setattr(args, opt_name, probe) - continue - if not os.path.exists(getattr(args, opt_name)): - raise RuntimeError( - f"{opt_name.replace('_', '-')} value {arg} not found" - ) - - def main(): if sys.platform not in ["cygwin", "win32", "linux"]: - _err(f"{sys.platform} not supported") + err(f"{sys.platform} not supported") args = parse_args() setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) build_dir = os.path.join(setup_dir, args.build_dir) - resolve_compilers(args) + c_compiler, cxx_compiler, compiler_root = resolve_compilers( + args.oneapi, args.c_compiler, args.cxx_compiler, args.compiler_root + ) # clean build dir if --clean set - if args.clean and os.path.exists(build_dir): - print(f"[build_locally] Cleaning build directory: {build_dir}") - shutil.rmtree(build_dir) - - env = os.environ.copy() - - # ignore pre-existing CMAKE_ARGS for determinism in build driver - if "CMAKE_ARGS" in env and env["CMAKE_ARGS"].strip(): - _warn("Ignoring pre-existing CMAKE_ARGS in environment") - del env["CMAKE_ARGS"] - - cmake_defs = [] - - # handle architecture conflicts - if args.target_hip is not None and not args.target_hip.strip(): - _err("--target-hip requires an explicit architecture") + if args.clean: + clean_build_dir(build_dir) if args.no_level_zero and args.target_level_zero: - _err("Cannot combine --no-level-zero and --target-level-zero") - - # CUDA/HIP targets - if args.target_cuda: - cmake_defs.append(f"-DDPCTL_TARGET_CUDA={args.target_cuda}") - if args.target_hip: - cmake_defs.append(f"-DDPCTL_TARGET_HIP={args.target_hip}") + err("Cannot combine --no-level-zero and --target-level-zero") # Level Zero state (on unless explicitly disabled) if args.no_level_zero: @@ -225,54 +165,48 @@ def main(): level_zero_enabled = True else: level_zero_enabled = True - cmake_defs.append( - "-DDPCTL_ENABLE_L0_PROGRAM_CREATION=" - f"{'ON' if level_zero_enabled else 'OFF'}" + + cmake_args = make_cmake_args( + build_type=args.build_type, + c_compiler=c_compiler, + cxx_compiler=cxx_compiler, + level_zero=level_zero_enabled, + glog=args.glog, + generator=args.generator, + verbose=args.verbose, + other_opts=args.cmake_opts, ) - # compilers and generator - if args.c_compiler: - cmake_defs.append(f"-DCMAKE_C_COMPILER:PATH={args.c_compiler}") - if args.cxx_compiler: - cmake_defs.append(f"-DCMAKE_CXX_COMPILER:PATH={args.cxx_compiler}") - if args.generator: - cmake_defs.append(f"-G{args.generator}") + # handle architecture conflicts + if args.target_hip is not None and not args.target_hip.strip(): + err("--target-hip requires an explicit architecture") + + # CUDA/HIP targets + if args.target_cuda: + cmake_args += f" -DDPCTL_TARGET_CUDA={args.target_cuda}" + if args.target_hip: + cmake_args += f" -DDPCTL_TARGET_HIP={args.target_hip}" - cmake_defs.append( - f"-DDPCTL_ENABLE_GLOG:BOOL={'ON' if args.glog else 'OFF'}" + cmake_args += ( + " -DDPCTL_ENABLE_L0_PROGRAM_CREATION=" + f"{'ON' if level_zero_enabled else 'OFF'}" ) - cmake_defs.append(f"-DCMAKE_BUILD_TYPE={args.build_type}") - if args.verbose: - cmake_defs.append("-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON") - if args.cmake_opts: - cmake_defs.extend(args.cmake_opts.split()) + env = os.environ.copy() - env["CMAKE_ARGS"] = " ".join(cmake_defs) - print(f"[build_locally] CMake args:\n {' '.join(cmake_defs)}") + # ignore pre-existing CMAKE_ARGS for determinism in build driver + if "CMAKE_ARGS" in env and env["CMAKE_ARGS"].strip(): + warn("Ignoring pre-existing CMAKE_ARGS in environment") + del env["CMAKE_ARGS"] + + env["CMAKE_ARGS"] = cmake_args + print(f"[build_locally] CMake args:\n {cmake_args}") print("[build_locally] Building extensions in-place...") - run( - [sys.executable, "setup.py", "build_ext", "--inplace"], - env=env, - cwd=setup_dir, - ) + build_extension(setup_dir, env) if not args.skip_editable: - print("[build_locally] Installing dpctl in editable mode") - run( - [ - sys.executable, - "-m", - "pip", - "install", - "-e", - ".", - "--no-build-isolation", - ], - env=env, - cwd=setup_dir, - ) + install_editable(setup_dir, env) else: print("[build_locally] Skipping editable install (--skip-editable)") From 989df1b08fba87606f7d4939225b3f1ed35be856 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Wed, 15 Oct 2025 08:40:26 -0700 Subject: [PATCH 04/17] remove --build-dir option and fix --clean option --- scripts/_build_helper.py | 11 ++++++----- scripts/build_locally.py | 10 ++-------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/scripts/_build_helper.py b/scripts/_build_helper.py index ede078a60a..ad1e5a2a61 100644 --- a/scripts/_build_helper.py +++ b/scripts/_build_helper.py @@ -1,6 +1,6 @@ # Data Parallel Control (dpctl) # -# Copyright 2020-2025 Intel Corporation +# Copyright 2025 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -121,7 +121,8 @@ def install_editable(setup_dir, env): ) -def clean_build_dir(build_dir): - if os.path.exists(build_dir): - print(f"Cleaning build directory: {build_dir}") - shutil.rmtree(build_dir) +def clean_build_dir(setup_dir): + target = os.path.join(setup_dir or os.getcwd(), "_skbuild") + if os.path.exists(target): + print(f"Cleaning build directory: {target}") + shutil.rmtree(target) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index ec1fde44ca..b8edcb0910 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -121,14 +121,9 @@ def parse_args(): help="Enable HIP backend. Architecture required to be specified.", ) - p.add_argument( - "--build-dir", - default="build", - help="CMake build directory (default: build)", - ) p.add_argument( "--clean", - action="store_false", + action="store_true", help="Remove build dir before rebuild (default: False)", ) p.add_argument( @@ -145,7 +140,6 @@ def main(): err(f"{sys.platform} not supported") args = parse_args() setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - build_dir = os.path.join(setup_dir, args.build_dir) c_compiler, cxx_compiler, compiler_root = resolve_compilers( args.oneapi, args.c_compiler, args.cxx_compiler, args.compiler_root @@ -153,7 +147,7 @@ def main(): # clean build dir if --clean set if args.clean: - clean_build_dir(build_dir) + clean_build_dir(setup_dir) if args.no_level_zero and args.target_level_zero: err("Cannot combine --no-level-zero and --target-level-zero") From 0648317da2776e335f0497d6bd430d97cc1df80b Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Wed, 15 Oct 2025 14:00:00 -0700 Subject: [PATCH 05/17] do not return compiler root unnecessarily from resolve_compilers --- scripts/_build_helper.py | 4 ++-- scripts/build_locally.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/_build_helper.py b/scripts/_build_helper.py index ad1e5a2a61..f20802fabb 100644 --- a/scripts/_build_helper.py +++ b/scripts/_build_helper.py @@ -43,7 +43,7 @@ def resolve_compilers( if oneapi or ( c_compiler is None and cxx_compiler is None and compiler_root is None ): - return "icx", ("icpx" if is_linux else "icx"), None + return "icx", ("icpx" if is_linux else "icx") if not compiler_root or not os.path.exists(compiler_root): raise RuntimeError( @@ -66,7 +66,7 @@ def resolve_compilers( ) if not os.path.exists(path): raise RuntimeError(f"{opt_name} value {name} not found") - return c_compiler, cxx_compiler, compiler_root + return c_compiler, cxx_compiler def make_cmake_args( diff --git a/scripts/build_locally.py b/scripts/build_locally.py index b8edcb0910..5bfb487e4d 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -141,7 +141,7 @@ def main(): args = parse_args() setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - c_compiler, cxx_compiler, compiler_root = resolve_compilers( + c_compiler, cxx_compiler = resolve_compilers( args.oneapi, args.c_compiler, args.cxx_compiler, args.compiler_root ) From a4d90be2c48b228002851c93f6d56be7a114c3a6 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Mon, 20 Oct 2025 17:19:02 -0700 Subject: [PATCH 06/17] update gen_coverage script to align with build_locally --- scripts/gen_coverage.py | 390 ++++++++++++++++++++-------------------- 1 file changed, 199 insertions(+), 191 deletions(-) diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index bdc2bd931c..06621be463 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -14,66 +14,172 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import os import re import subprocess import sys import sysconfig +# add scripts dir to Python path so we can import _build_helper +sys.path.insert(0, os.path.abspath("scripts")) -def run( - use_oneapi=True, - c_compiler=None, - cxx_compiler=None, - level_zero=True, - compiler_root=None, - run_pytest=False, - bin_llvm=None, - gtest_config=None, - verbose=False, -): - IS_LIN = False - - if "linux" in sys.platform: - IS_LIN = True - elif sys.platform in ["win32", "cygwin"]: - pass - else: - assert False, sys.platform + " not supported" +from _build_helper import ( # noqa: E402 + build_extension, + clean_build_dir, + err, + install_editable, + make_cmake_args, + resolve_compilers, + run, + warn, +) - if not IS_LIN: - raise RuntimeError( - "This scripts only supports coverage collection on Linux" - ) + +def parse_args(): + p = argparse.ArgumentParser(description="Build dpctl and generate coverage") + + p.add_argument( + "--c-compiler", default=None, help="Path or name of C compiler" + ) + p.add_argument( + "--cxx-compiler", default=None, help="Path or name of C++ compiler" + ) + p.add_argument( + "--compiler-root", + type=str, + default=None, + help="Path to compiler installation root", + ) + p.add_argument( + "--oneapi", + dest="oneapi", + action="store_true", + help="Use default oneAPI compiler layout", + ) + + p.add_argument( + "--verbose", + dest="verbose", + action="store_true", + help="Enable verbose makefile output", + ) + + p.add_argument( + "--no-level-zero", + dest="no_level_zero", + action="store_true", + default=False, + help="Disable Level Zero backend (deprecated: use --target-level-zero " + "OFF)", + ) + p.add_argument( + "--target-level-zero", + action="store_true", + help="Enable Level Zero backend explicitly", + ) + + p.add_argument( + "--cmake-opts", + type=str, + default="", + help="Additional options to pass directly to CMake", + ) + + p.add_argument( + "--gtest-config", + help="Path to GTestConfig.cmake file for a custom GTest installation", + ) + p.add_argument( + "--bin-llvm", + help="Path to folder where llvm-cov/llvm-profdata can be found", + ) + p.add_argument("--skip-pytest", dest="run_pytest", action="store_false") + p.add_argument( + "--clean", + action="store_true", + help="Remove build dir before rebuild (default: False)", + ) + + return p.parse_args() + + +def find_objects(setup_dir): + objects = [] + sfx_regexp = sysconfig.get_config_var("EXT_SUFFIX").replace(".", r"\.") + regexp1 = re.compile(r"^_tensor_.*impl" + sfx_regexp) + regexp2 = re.compile(r"^^_device_queries" + sfx_regexp) + + def is_py_ext(fn): + return re.match(regexp1, fn) or re.match(regexp2, fn) + + for root, _, files in os.walk(os.path.join(setup_dir, "dpctl")): + for file in files: + if not file.endswith(".so"): + continue + if is_py_ext(file) or "DPCTLSyclInterface" in file: + objects.extend(["-object", os.path.join(root, file)]) + print("[gen_coverage] Using objects:", objects) + return objects + + +def main(): + is_linux = "linux" in sys.platform + if not is_linux: + err(f"{sys.platform} not supported") + args = parse_args() setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - cmake_args = [ - sys.executable, - "setup.py", - "develop", - "--build-type=Coverage", - "--generator=Ninja", - "--", - "-DCMAKE_C_COMPILER:PATH=" + c_compiler, - "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, - "-DDPCTL_ENABLE_L0_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), - "-DDPCTL_GENERATE_COVERAGE=ON", - "-DDPCTL_BUILD_CAPI_TESTS=ON", - "-DDPCTL_COVERAGE_REPORT_OUTPUT_DIR=" + setup_dir, - ] - env = dict() - if bin_llvm: - env = { - "PATH": ":".join((os.environ.get("PATH", ""), bin_llvm)), - "LLVM_TOOLS_HOME": bin_llvm, - } - env.update({k: v for k, v in os.environ.items() if k != "PATH"}) - if gtest_config: - cmake_args += ["-DCMAKE_PREFIX_PATH=" + gtest_config] - if verbose: - cmake_args += [ - "-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON", - ] - subprocess.check_call(cmake_args, shell=False, cwd=setup_dir, env=env) + + c_compiler, cxx_compiler = resolve_compilers( + args.oneapi, args.c_compiler, args.cxx_compiler, args.compiler_root + ) + + if args.clean: + clean_build_dir(setup_dir) + + if args.no_level_zero and args.target_level_zero: + err("Cannot combine --no-level-zero and --target-level-zero") + + # Level Zero state (on unless explicitly disabled) + if args.no_level_zero: + level_zero_enabled = False + elif args.target_level_zero: + level_zero_enabled = True + else: + level_zero_enabled = True + + cmake_args = make_cmake_args( + build_type="Coverage", + c_compiler=c_compiler, + cxx_compiler=cxx_compiler, + level_zero=level_zero_enabled, + verbose=args.verbose, + ) + + cmake_args += " -DDPCTL_GENERATE_COVERAGE=ON" + cmake_args += " -DDPCTL_BUILD_CAPI_TESTS=ON" + cmake_args += f" -DDPCTL_COVERAGE_REPORT_OUTPUT={setup_dir}" + + if args.gtest_config: + cmake_args += " -DCMAKE_PREFIX_PATH={args.gtest_config}" + + env = os.environ.copy() + + if "CMAKE_ARGS" in env and env["CMAKE_ARGS"].strip(): + warn("Ignoring pre-existing CMAKE_ARGS in environment") + del env["CMAKE_ARGS"] + + env["CMAKE_ARGS"] = cmake_args + + if args.bin_llvm: + env["PATH"] = ":".join((env.get("PATH", ""), args.bin_llvm)) + env["LLVM_TOOLS_HOME"] = args.bin_llvm + + print(f"[gen_coverage] Using CMake args:\n {env['CMAKE_ARGS']}") + + build_extension(setup_dir, env) + install_editable(setup_dir, env) + cmake_build_dir = ( subprocess.check_output( ["find", "_skbuild", "-name", "cmake-build"], cwd=setup_dir @@ -81,13 +187,16 @@ def run( .decode("utf-8") .strip("\n") ) - subprocess.check_call( + print(f"[gen_coverage] Found CMake build dir: {cmake_build_dir}") + + run( ["cmake", "--build", ".", "--target", "llvm-cov-report"], cwd=cmake_build_dir, ) - env["LLVM_PROFILE_FILE"] = "dpctl_pytest.profraw" - subprocess.check_call( - [ + + if args.run_pytest: + env["LLVM_PROFILE_FILE"] = "dpctl_pytest.profraw" + pytest_cmd = [ "pytest", "-q", "-ra", @@ -103,147 +212,46 @@ def run( "-vv", "--ignore=dpctl/tensor/libtensor/tests", "--no-sycl-interface-test", - ], - cwd=setup_dir, - shell=False, - env=env, - ) - - def find_objects(): - import os - - objects = [] - sfx_regexp = sysconfig.get_config_var("EXT_SUFFIX").replace(".", r"\.") - regexp1 = re.compile(r"^_tensor_.*impl" + sfx_regexp) - regexp2 = re.compile(r"^^_device_queries" + sfx_regexp) - - def is_py_ext(fn): - return re.match(regexp1, fn) or re.match(regexp2, fn) - - for root, _, files in os.walk("dpctl"): - for file in files: - if not file.endswith(".so"): - continue - if is_py_ext(file) or file.find("DPCTLSyclInterface") != -1: - objects.extend(["-object", os.path.join(root, file)]) - print("Using objects: ", objects) - return objects - - objects = find_objects() - instr_profile_fn = "dpctl_pytest.profdata" - # generate instrumentation profile data - subprocess.check_call( - [ - os.path.join(bin_llvm, "llvm-profdata"), - "merge", - "-sparse", - env["LLVM_PROFILE_FILE"], - "-o", - instr_profile_fn, ] - ) - # export lcov - with open("dpctl_pytest.lcov", "w") as fh: - subprocess.check_call( + run(pytest_cmd, env=env, cwd=setup_dir) + + objects = find_objects(setup_dir) + instr_profile_fn = "dpctl_pytest.profdata" + + run( [ - os.path.join(bin_llvm, "llvm-cov"), - "export", - "-format=lcov", - "-ignore-filename-regex=/tmp/icpx*", - "-instr-profile=" + instr_profile_fn, + os.path.join(args.bin_llvm or "", "llvm-profdata"), + "merge", + "-sparse", + env["LLVM_PROFILE_FILE"], + "-o", + instr_profile_fn, ] - + objects, - stdout=fh, ) + with open("dpctl_pytest.lcov", "w") as fh: + subprocess.check_call( + [ + os.path.join(args.bin_llvm or "", "llvm-cov"), + "export", + "-format=lcov", + "-ignore-filename-regex=/tmp/icpx*", + f"-instr-profile={instr_profile_fn}", + ] + + objects, + cwd=setup_dir, + env=env, + stdout=fh, + ) + print("[gen_coverage] Coverage export complete: dpctl_pytest.lcov") + else: + print( + "[gen_coverage] Skipping pytest and coverage collection " + "(--skip-pytest)" + ) -if __name__ == "__main__": - import argparse + print("[gen_coverage] Done") - parser = argparse.ArgumentParser( - description="Driver to build dpctl and generate coverage" - ) - driver = parser.add_argument_group(title="Coverage driver arguments") - driver.add_argument("--c-compiler", help="Name of C compiler", default=None) - driver.add_argument( - "--cxx-compiler", help="Name of C++ compiler", default=None - ) - driver.add_argument( - "--not-oneapi", - help="Is one-API installation", - dest="oneapi", - action="store_false", - ) - driver.add_argument( - "--compiler-root", type=str, help="Path to compiler home directory" - ) - driver.add_argument( - "--no-level-zero", - help="Enable Level Zero support", - dest="level_zero", - action="store_false", - ) - driver.add_argument( - "--skip-pytest", - help="Run pytest and collect coverage", - dest="run_pytest", - action="store_false", - ) - driver.add_argument( - "--bin-llvm", help="Path to folder where llvm-cov can be found" - ) - driver.add_argument( - "--verbose", - help="Build using vebose makefile mode", - dest="verbose", - action="store_true", - ) - driver.add_argument( - "--gtest-config", - help="Path to the GTestConfig.cmake file to locate a " - + "custom GTest installation.", - ) - args = parser.parse_args() - - if args.oneapi: - args.c_compiler = "icx" - args.cxx_compiler = "icpx" - args.compiler_root = None - icx_path = subprocess.check_output(["which", "icx"]) - bin_dir = os.path.dirname(icx_path) - compiler_dir = os.path.join(bin_dir.decode("utf-8"), "compiler") - if os.path.exists(compiler_dir): - args.bin_llvm = os.path.join(bin_dir.decode("utf-8"), "compiler") - else: - bin_dir = os.path.dirname(bin_dir) - args.bin_llvm = os.path.join(bin_dir.decode("utf-8"), "bin-llvm") - assert os.path.exists(args.bin_llvm) - else: - args_to_validate = [ - "c_compiler", - "cxx_compiler", - "compiler_root", - "bin_llvm", - ] - for p in args_to_validate: - arg = getattr(args, p, None) - if not isinstance(arg, str): - opt_name = p.replace("_", "-") - raise RuntimeError( - f"Option {opt_name} must be provided is " - "using non-default DPC++ layout" - ) - if not os.path.exists(arg): - raise RuntimeError(f"Path {arg} must exist") - run( - use_oneapi=args.oneapi, - c_compiler=args.c_compiler, - cxx_compiler=args.cxx_compiler, - level_zero=args.level_zero, - compiler_root=args.compiler_root, - run_pytest=args.run_pytest, - bin_llvm=args.bin_llvm, - gtest_config=args.gtest_config, - verbose=args.verbose, - ) +if __name__ == "__main__": + main() From 7b536236f88d87db974e1aa79cca70005c07192f Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Mon, 27 Oct 2025 23:39:33 -0700 Subject: [PATCH 07/17] pass explicit CMake variables for coverage tool paths --- scripts/gen_coverage.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 06621be463..4c07afd9a6 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -169,11 +169,23 @@ def main(): warn("Ignoring pre-existing CMAKE_ARGS in environment") del env["CMAKE_ARGS"] - env["CMAKE_ARGS"] = cmake_args - if args.bin_llvm: - env["PATH"] = ":".join((env.get("PATH", ""), args.bin_llvm)) + llvm_cov = os.path.join(args.bin_llvm, "llvm-cov") + llvm_profdata = os.path.join(args.bin_llvm, "llvm-profdata") + env = os.environ.copy() + if not (os.path.isfile(llvm_cov) and os.access(llvm_cov, os.X_OK)): + err(f"Cannot find executable llvm-cov in {args.bin_llvm}") + if not ( + os.path.isfile(llvm_profdata) and os.access(llvm_profdata, os.X_OK) + ): + err(f"Cannot find executable llvm-profdata in {args.bin_llvm}") + env["PATH"] = f"{args.bin_llvm}:{env.get('PATH', '')}" env["LLVM_TOOLS_HOME"] = args.bin_llvm + cmake_args += f" -DLLVM_TOOLS_HOME={args.bin_llvm}" + cmake_args += f" -DLLVM_PROFDATA={llvm_profdata}" + cmake_args += f" -DLLVM_COV={llvm_cov}" + + env["CMAKE_ARGS"] = cmake_args print(f"[gen_coverage] Using CMake args:\n {env['CMAKE_ARGS']}") From 72ca5a04ac0ed724534a91722286c489a2d8739a Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 28 Oct 2025 01:00:55 -0700 Subject: [PATCH 08/17] replicate old script behavior when calling setup.py --- scripts/_build_helper.py | 18 +++++++++++------- scripts/build_locally.py | 10 +++++++--- scripts/gen_coverage.py | 37 ++++++++++++++++++++----------------- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/scripts/_build_helper.py b/scripts/_build_helper.py index f20802fabb..c296629b95 100644 --- a/scripts/_build_helper.py +++ b/scripts/_build_helper.py @@ -70,25 +70,20 @@ def resolve_compilers( def make_cmake_args( - build_type="Release", c_compiler=None, cxx_compiler=None, level_zero=True, glog=False, - generator=None, verbose=False, other_opts="", ): args = [ - f"-DCMAKE_BUILD_TYPE={build_type}", f"-DCMAKE_C_COMPILER:PATH={c_compiler}" if c_compiler else "", f"-DCMAKE_CXX_COMPILER:PATH={cxx_compiler}" if cxx_compiler else "", f"-DDPCTL_ENABLE_L0_PROGRAM_CREATION={'ON' if level_zero else 'OFF'}", f"-DDPTL_ENABLE_GLOG:BOOL={'ON' if glog else 'OFF'}", ] - if generator: - args.append(f"-G{generator}") if verbose: args.append("-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON") if other_opts: @@ -97,9 +92,18 @@ def make_cmake_args( return " ".join(filter(None, args)) -def build_extension(setup_dir, env): +def build_extension( + setup_dir, env, cmake_executable=None, generator=None, build_type=None +): + cmd = [sys.executable, "setup.py", "build_ext", "--inplace"] + if cmake_executable: + cmd.append(f"--cmake-executable={cmake_executable}") + if generator: + cmd.append(f"--generator={generator}") + if build_type: + cmd.append(f"--build-type={build_type}") run( - [sys.executable, "setup.py", "build_ext", "--inplace"], + cmd, env=env, cwd=setup_dir, ) diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 5bfb487e4d..4c991f74b3 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -161,12 +161,10 @@ def main(): level_zero_enabled = True cmake_args = make_cmake_args( - build_type=args.build_type, c_compiler=c_compiler, cxx_compiler=cxx_compiler, level_zero=level_zero_enabled, glog=args.glog, - generator=args.generator, verbose=args.verbose, other_opts=args.cmake_opts, ) @@ -198,7 +196,13 @@ def main(): print("[build_locally] Building extensions in-place...") - build_extension(setup_dir, env) + build_extension( + setup_dir, + env, + cmake_executable=args.cmake_executable, + generator=args.generator, + build_type=args.build_type, + ) if not args.skip_editable: install_editable(setup_dir, env) else: diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 4c07afd9a6..c3de23bf73 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -79,6 +79,16 @@ def parse_args(): help="Enable Level Zero backend explicitly", ) + p.add_argument( + "--generator", type=str, default="Ninja", help="CMake generator" + ) + p.add_argument( + "--cmake-executable", + type=str, + default=None, + help="Path to CMake executable used by build", + ) + p.add_argument( "--cmake-opts", type=str, @@ -149,7 +159,6 @@ def main(): level_zero_enabled = True cmake_args = make_cmake_args( - build_type="Coverage", c_compiler=c_compiler, cxx_compiler=cxx_compiler, level_zero=level_zero_enabled, @@ -169,27 +178,21 @@ def main(): warn("Ignoring pre-existing CMAKE_ARGS in environment") del env["CMAKE_ARGS"] + env["CMAKE_ARGS"] = cmake_args + if args.bin_llvm: - llvm_cov = os.path.join(args.bin_llvm, "llvm-cov") - llvm_profdata = os.path.join(args.bin_llvm, "llvm-profdata") - env = os.environ.copy() - if not (os.path.isfile(llvm_cov) and os.access(llvm_cov, os.X_OK)): - err(f"Cannot find executable llvm-cov in {args.bin_llvm}") - if not ( - os.path.isfile(llvm_profdata) and os.access(llvm_profdata, os.X_OK) - ): - err(f"Cannot find executable llvm-profdata in {args.bin_llvm}") - env["PATH"] = f"{args.bin_llvm}:{env.get('PATH', '')}" + env["PATH"] = ":".join((env.get("PATH", ""), args.bin_llvm)) env["LLVM_TOOLS_HOME"] = args.bin_llvm - cmake_args += f" -DLLVM_TOOLS_HOME={args.bin_llvm}" - cmake_args += f" -DLLVM_PROFDATA={llvm_profdata}" - cmake_args += f" -DLLVM_COV={llvm_cov}" - - env["CMAKE_ARGS"] = cmake_args print(f"[gen_coverage] Using CMake args:\n {env['CMAKE_ARGS']}") - build_extension(setup_dir, env) + build_extension( + setup_dir, + env, + cmake_executable=args.cmake_executable, + generator=args.generator, + build_type="Coverage", + ) install_editable(setup_dir, env) cmake_build_dir = ( From 80af656bbe65409d1c771b5620e42f92780f6d68 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 28 Oct 2025 01:14:26 -0700 Subject: [PATCH 09/17] attempt setting cmake_args for LLVMCov_EXE --- scripts/gen_coverage.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index c3de23bf73..6d7619fd4a 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -178,14 +178,21 @@ def main(): warn("Ignoring pre-existing CMAKE_ARGS in environment") del env["CMAKE_ARGS"] - env["CMAKE_ARGS"] = cmake_args - if args.bin_llvm: env["PATH"] = ":".join((env.get("PATH", ""), args.bin_llvm)) env["LLVM_TOOLS_HOME"] = args.bin_llvm + llvm_profdata = os.path.join(args.bin_llvm, "llvm-profdata") + llvm_cov = os.path.join(args.bin_llvm, "llvm-cov") + cmake_args += f" -DLLVM_TOOLS_HOME={args.bin_llvm}" + cmake_args += f" -DLLVM_PROFDATA={llvm_profdata}" + cmake_args += f" -DLLVM_COV={llvm_cov}" + # Add LLVMCov_EXE for CMake find_package(LLVMCov) + cmake_args += f" -DLLVMCov_EXE={llvm_cov}" print(f"[gen_coverage] Using CMake args:\n {env['CMAKE_ARGS']}") + env["CMAKE_ARGS"] = cmake_args + build_extension( setup_dir, env, From 527de4f818072770364cfb401c4e92430ff91e45 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 28 Oct 2025 08:33:00 -0700 Subject: [PATCH 10/17] resolve bin_llvm from default compiler layout when not provided --- scripts/_build_helper.py | 34 ---------------- scripts/build_locally.py | 35 ++++++++++++++++- scripts/gen_coverage.py | 83 ++++++++++++++++++++++++++++++++-------- 3 files changed, 102 insertions(+), 50 deletions(-) diff --git a/scripts/_build_helper.py b/scripts/_build_helper.py index c296629b95..e61cdda05e 100644 --- a/scripts/_build_helper.py +++ b/scripts/_build_helper.py @@ -35,40 +35,6 @@ def err(msg: str): print(f"[build_locally][error] {msg}", file=sys.stderr) -def resolve_compilers( - oneapi: bool, c_compiler: str, cxx_compiler: str, compiler_root: str -): - is_linux = "linux" in sys.platform - - if oneapi or ( - c_compiler is None and cxx_compiler is None and compiler_root is None - ): - return "icx", ("icpx" if is_linux else "icx") - - if not compiler_root or not os.path.exists(compiler_root): - raise RuntimeError( - "--compiler-root option must be set when using non-default DPC++ " - "layout" - ) - - # default values - if c_compiler is None: - c_compiler = "icx" - if cxx_compiler is None: - cxx_compiler = "icpx" if is_linux else "icx" - - for name, opt_name in ( - (c_compiler, "--c-compiler"), - (cxx_compiler, "--cxx-compiler"), - ): - path = ( - name if os.path.exists(name) else os.path.join(compiler_root, name) - ) - if not os.path.exists(path): - raise RuntimeError(f"{opt_name} value {name} not found") - return c_compiler, cxx_compiler - - def make_cmake_args( c_compiler=None, cxx_compiler=None, diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 4c991f74b3..908c775e99 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -27,11 +27,44 @@ err, install_editable, make_cmake_args, - resolve_compilers, warn, ) +def resolve_compilers( + oneapi: bool, c_compiler: str, cxx_compiler: str, compiler_root: str +): + is_linux = "linux" in sys.platform + + if oneapi or ( + c_compiler is None and cxx_compiler is None and compiler_root is None + ): + return "icx", ("icpx" if is_linux else "icx") + + if not compiler_root or not os.path.exists(compiler_root): + raise RuntimeError( + "--compiler-root option must be set when using non-default DPC++ " + "layout" + ) + + # default values + if c_compiler is None: + c_compiler = "icx" + if cxx_compiler is None: + cxx_compiler = "icpx" if is_linux else "icx" + + for name, opt_name in ( + (c_compiler, "--c-compiler"), + (cxx_compiler, "--cxx-compiler"), + ): + path = ( + name if os.path.exists(name) else os.path.join(compiler_root, name) + ) + if not os.path.exists(path): + raise RuntimeError(f"{opt_name} value {name} not found") + return c_compiler, cxx_compiler + + def parse_args(): p = argparse.ArgumentParser(description="Local dpctl build driver") diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 6d7619fd4a..49f61cae51 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -30,12 +30,68 @@ err, install_editable, make_cmake_args, - resolve_compilers, run, warn, ) +def find_bin_llvm(compiler_name): + icx_path = subprocess.check_output(["which", compiler_name]) + bin_dir = os.path.dirname(icx_path) + compiler_dir = os.path.join(bin_dir.decode("utf-8"), "compiler") + if os.path.exists(compiler_dir): + bin_llvm = compiler_dir + else: + bin_dir = os.path.dirname(bin_dir) + bin_llvm = os.path.join(bin_dir.decode("utf-8"), "bin-llvm") + assert os.path.exists(bin_llvm) + return bin_llvm + + +def resolve_compilers( + oneapi: bool, + c_compiler: str, + cxx_compiler: str, + compiler_root: str, + bin_llvm: str = None, +): + is_linux = "linux" in sys.platform + + if oneapi or ( + c_compiler is None + and cxx_compiler is None + and compiler_root is None + and bin_llvm is None + ): + return "icx", ("icpx" if is_linux else "icx"), find_bin_llvm("icx") + + if not compiler_root or not os.path.exists(compiler_root): + raise RuntimeError( + "--compiler-root option must be set when using non-default DPC++ " + "layout" + ) + + # default values + if c_compiler is None: + c_compiler = "icx" + if cxx_compiler is None: + cxx_compiler = "icpx" if is_linux else "icx" + if bin_llvm is None: + bin_llvm = find_bin_llvm(c_compiler) + + for name, opt_name in ( + (c_compiler, "--c-compiler"), + (cxx_compiler, "--cxx-compiler"), + (bin_llvm, "--bin-llvm"), + ): + path = ( + name if os.path.exists(name) else os.path.join(compiler_root, name) + ) + if not os.path.exists(path): + raise RuntimeError(f"{opt_name} value {name} not found") + return c_compiler, cxx_compiler, bin_llvm + + def parse_args(): p = argparse.ArgumentParser(description="Build dpctl and generate coverage") @@ -140,8 +196,12 @@ def main(): args = parse_args() setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - c_compiler, cxx_compiler = resolve_compilers( - args.oneapi, args.c_compiler, args.cxx_compiler, args.compiler_root + c_compiler, cxx_compiler, bin_llvm = resolve_compilers( + args.oneapi, + args.c_compiler, + args.cxx_compiler, + args.compiler_root, + args.bin_llvm, ) if args.clean: @@ -178,21 +238,14 @@ def main(): warn("Ignoring pre-existing CMAKE_ARGS in environment") del env["CMAKE_ARGS"] - if args.bin_llvm: - env["PATH"] = ":".join((env.get("PATH", ""), args.bin_llvm)) - env["LLVM_TOOLS_HOME"] = args.bin_llvm - llvm_profdata = os.path.join(args.bin_llvm, "llvm-profdata") - llvm_cov = os.path.join(args.bin_llvm, "llvm-cov") - cmake_args += f" -DLLVM_TOOLS_HOME={args.bin_llvm}" - cmake_args += f" -DLLVM_PROFDATA={llvm_profdata}" - cmake_args += f" -DLLVM_COV={llvm_cov}" - # Add LLVMCov_EXE for CMake find_package(LLVMCov) - cmake_args += f" -DLLVMCov_EXE={llvm_cov}" - - print(f"[gen_coverage] Using CMake args:\n {env['CMAKE_ARGS']}") + if bin_llvm: + env["PATH"] = ":".join((env.get("PATH", ""), bin_llvm)) + env["LLVM_TOOLS_HOME"] = bin_llvm env["CMAKE_ARGS"] = cmake_args + print(f"[gen_coverage] Using CMake args:\n {env['CMAKE_ARGS']}") + build_extension( setup_dir, env, From 897300617b78c23da55c9705bc8d6c026b27f380 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 28 Oct 2025 09:43:44 -0700 Subject: [PATCH 11/17] use bin_llvm when running llvm-cov and llvm-profdata --- scripts/gen_coverage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 49f61cae51..94de41a781 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -295,7 +295,7 @@ def main(): run( [ - os.path.join(args.bin_llvm or "", "llvm-profdata"), + os.path.join(bin_llvm, "llvm-profdata"), "merge", "-sparse", env["LLVM_PROFILE_FILE"], @@ -307,7 +307,7 @@ def main(): with open("dpctl_pytest.lcov", "w") as fh: subprocess.check_call( [ - os.path.join(args.bin_llvm or "", "llvm-cov"), + os.path.join(bin_llvm, "llvm-cov"), "export", "-format=lcov", "-ignore-filename-regex=/tmp/icpx*", From 50394b1f830764d8cef200cfa50a9398b9a15258 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 28 Oct 2025 09:53:18 -0700 Subject: [PATCH 12/17] keep find_objects defined within main --- scripts/gen_coverage.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 94de41a781..d14b4221d8 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -170,25 +170,6 @@ def parse_args(): return p.parse_args() -def find_objects(setup_dir): - objects = [] - sfx_regexp = sysconfig.get_config_var("EXT_SUFFIX").replace(".", r"\.") - regexp1 = re.compile(r"^_tensor_.*impl" + sfx_regexp) - regexp2 = re.compile(r"^^_device_queries" + sfx_regexp) - - def is_py_ext(fn): - return re.match(regexp1, fn) or re.match(regexp2, fn) - - for root, _, files in os.walk(os.path.join(setup_dir, "dpctl")): - for file in files: - if not file.endswith(".so"): - continue - if is_py_ext(file) or "DPCTLSyclInterface" in file: - objects.extend(["-object", os.path.join(root, file)]) - print("[gen_coverage] Using objects:", objects) - return objects - - def main(): is_linux = "linux" in sys.platform if not is_linux: @@ -290,7 +271,27 @@ def main(): ] run(pytest_cmd, env=env, cwd=setup_dir) - objects = find_objects(setup_dir) + def find_objects(): + objects = [] + sfx_regexp = sysconfig.get_config_var("EXT_SUFFIX").replace( + ".", r"\." + ) + regexp1 = re.compile(r"^_tensor_.*impl" + sfx_regexp) + regexp2 = re.compile(r"^^_device_queries" + sfx_regexp) + + def is_py_ext(fn): + return re.match(regexp1, fn) or re.match(regexp2, fn) + + for root, _, files in os.walk("dpctl"): + for file in files: + if not file.endswith(".so"): + continue + if is_py_ext(file) or "DPCTLSyclInterface" in file: + objects.extend(["-object", os.path.join(root, file)]) + print("Using objects: ", objects) + return objects + + objects = find_objects() instr_profile_fn = "dpctl_pytest.profdata" run( From f327741ddad6f9286baf36ba22c0a2d1f4f47a0a Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 28 Oct 2025 10:55:29 -0700 Subject: [PATCH 13/17] generalize err and warn utilities for different build scripts --- scripts/_build_helper.py | 8 ++++---- scripts/build_locally.py | 11 +++++++---- scripts/gen_coverage.py | 9 ++++++--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/scripts/_build_helper.py b/scripts/_build_helper.py index e61cdda05e..cc334d3427 100644 --- a/scripts/_build_helper.py +++ b/scripts/_build_helper.py @@ -27,12 +27,12 @@ def run(cmd, env=None, cwd=None): ) -def warn(msg: str): - print(f"[build_locally][error] {msg}", file=sys.stderr) +def warn(msg: str, script: str): + print(f"[{script}][warning] {msg}", file=sys.stderr) -def err(msg: str): - print(f"[build_locally][error] {msg}", file=sys.stderr) +def err(msg: str, script: str): + print(f"[{script}][error] {msg}", file=sys.stderr) def make_cmake_args( diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 908c775e99..253b9a2dd0 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -170,7 +170,7 @@ def parse_args(): def main(): if sys.platform not in ["cygwin", "win32", "linux"]: - err(f"{sys.platform} not supported") + err(f"{sys.platform} not supported", "build_locally") args = parse_args() setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -183,7 +183,10 @@ def main(): clean_build_dir(setup_dir) if args.no_level_zero and args.target_level_zero: - err("Cannot combine --no-level-zero and --target-level-zero") + err( + "Cannot combine --no-level-zero and --target-level-zero", + "build_locally", + ) # Level Zero state (on unless explicitly disabled) if args.no_level_zero: @@ -204,7 +207,7 @@ def main(): # handle architecture conflicts if args.target_hip is not None and not args.target_hip.strip(): - err("--target-hip requires an explicit architecture") + err("--target-hip requires an explicit architecture", "build_locally") # CUDA/HIP targets if args.target_cuda: @@ -221,7 +224,7 @@ def main(): # ignore pre-existing CMAKE_ARGS for determinism in build driver if "CMAKE_ARGS" in env and env["CMAKE_ARGS"].strip(): - warn("Ignoring pre-existing CMAKE_ARGS in environment") + warn("Ignoring pre-existing CMAKE_ARGS in environment", "build_locally") del env["CMAKE_ARGS"] env["CMAKE_ARGS"] = cmake_args diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index d14b4221d8..17c438eed4 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -173,7 +173,7 @@ def parse_args(): def main(): is_linux = "linux" in sys.platform if not is_linux: - err(f"{sys.platform} not supported") + err(f"{sys.platform} not supported", "gen_coverage") args = parse_args() setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -189,7 +189,10 @@ def main(): clean_build_dir(setup_dir) if args.no_level_zero and args.target_level_zero: - err("Cannot combine --no-level-zero and --target-level-zero") + err( + "Cannot combine --no-level-zero and --target-level-zero", + "gen_coverage", + ) # Level Zero state (on unless explicitly disabled) if args.no_level_zero: @@ -216,7 +219,7 @@ def main(): env = os.environ.copy() if "CMAKE_ARGS" in env and env["CMAKE_ARGS"].strip(): - warn("Ignoring pre-existing CMAKE_ARGS in environment") + warn("Ignoring pre-existing CMAKE_ARGS in environment", "gen_coverage") del env["CMAKE_ARGS"] if bin_llvm: From 6c33bf09b91ff05bc7e4b6db070a8d7d86cdc70f Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 28 Oct 2025 14:53:30 -0700 Subject: [PATCH 14/17] use common resolve_compilers utility --- scripts/_build_helper.py | 75 ++++++++++++++++++++++++++++++++++------ scripts/build_locally.py | 37 ++------------------ scripts/gen_coverage.py | 74 ++++++++++++--------------------------- 3 files changed, 89 insertions(+), 97 deletions(-) diff --git a/scripts/_build_helper.py b/scripts/_build_helper.py index cc334d3427..6d3128ffbf 100644 --- a/scripts/_build_helper.py +++ b/scripts/_build_helper.py @@ -20,13 +20,64 @@ import sys -def run(cmd, env=None, cwd=None): +def resolve_compilers( + oneapi: bool, + c_compiler: str, + cxx_compiler: str, + compiler_root: str, +): + is_linux = "linux" in sys.platform + + if oneapi or ( + c_compiler is None and cxx_compiler is None and compiler_root is None + ): + return "icx", ("icpx" if is_linux else "icx") + + if ( + (c_compiler is None or not os.path.isabs(c_compiler)) + and (cxx_compiler is None or not os.path.isabs(cxx_compiler)) + and (not compiler_root or not os.path.exists(compiler_root)) + ): + raise RuntimeError( + "--compiler-root option must be set when using non-default DPC++ " + "layout unless absolute paths are provided for both compilers" + ) + + # default values + if c_compiler is None: + c_compiler = "icx" + if cxx_compiler is None: + cxx_compiler = "icpx" if is_linux else "icx" + + for name, opt_name in ( + (c_compiler, "--c-compiler"), + (cxx_compiler, "--cxx-compiler"), + ): + if os.path.isabs(name): + path = name + else: + path = os.path.join(compiler_root, name) + if not os.path.exists(path): + raise RuntimeError(f"{opt_name} value {name} not found") + return c_compiler, cxx_compiler + + +def run(cmd: list[str], env: dict[str, str] = None, cwd: str = None): print("+", " ".join(cmd)) subprocess.check_call( cmd, env=env or os.environ.copy(), cwd=cwd or os.getcwd() ) +def get_output(cmd: list[str], cwd: str = None): + print("+", " ".join(cmd)) + return ( + subprocess.check_output(cmd, cwd=cwd or os.getcwd()) + .decode("utf-8") + .strip("\n") + ) + + def warn(msg: str, script: str): print(f"[{script}][warning] {msg}", file=sys.stderr) @@ -36,12 +87,12 @@ def err(msg: str, script: str): def make_cmake_args( - c_compiler=None, - cxx_compiler=None, - level_zero=True, - glog=False, - verbose=False, - other_opts="", + c_compiler: str = None, + cxx_compiler: str = None, + level_zero: bool = True, + glog: bool = False, + verbose: bool = False, + other_opts: str = None, ): args = [ f"-DCMAKE_C_COMPILER:PATH={c_compiler}" if c_compiler else "", @@ -59,7 +110,11 @@ def make_cmake_args( def build_extension( - setup_dir, env, cmake_executable=None, generator=None, build_type=None + setup_dir: str, + env: dict[str, str], + cmake_executable: str = None, + generator: str = None, + build_type: str = None, ): cmd = [sys.executable, "setup.py", "build_ext", "--inplace"] if cmake_executable: @@ -75,7 +130,7 @@ def build_extension( ) -def install_editable(setup_dir, env): +def install_editable(setup_dir: str, env: dict[str, str]): run( [ sys.executable, @@ -91,7 +146,7 @@ def install_editable(setup_dir, env): ) -def clean_build_dir(setup_dir): +def clean_build_dir(setup_dir: str = None): target = os.path.join(setup_dir or os.getcwd(), "_skbuild") if os.path.exists(target): print(f"Cleaning build directory: {target}") diff --git a/scripts/build_locally.py b/scripts/build_locally.py index 253b9a2dd0..76c7c61eb5 100644 --- a/scripts/build_locally.py +++ b/scripts/build_locally.py @@ -27,44 +27,11 @@ err, install_editable, make_cmake_args, + resolve_compilers, warn, ) -def resolve_compilers( - oneapi: bool, c_compiler: str, cxx_compiler: str, compiler_root: str -): - is_linux = "linux" in sys.platform - - if oneapi or ( - c_compiler is None and cxx_compiler is None and compiler_root is None - ): - return "icx", ("icpx" if is_linux else "icx") - - if not compiler_root or not os.path.exists(compiler_root): - raise RuntimeError( - "--compiler-root option must be set when using non-default DPC++ " - "layout" - ) - - # default values - if c_compiler is None: - c_compiler = "icx" - if cxx_compiler is None: - cxx_compiler = "icpx" if is_linux else "icx" - - for name, opt_name in ( - (c_compiler, "--c-compiler"), - (cxx_compiler, "--cxx-compiler"), - ): - path = ( - name if os.path.exists(name) else os.path.join(compiler_root, name) - ) - if not os.path.exists(path): - raise RuntimeError(f"{opt_name} value {name} not found") - return c_compiler, cxx_compiler - - def parse_args(): p = argparse.ArgumentParser(description="Local dpctl build driver") @@ -157,7 +124,7 @@ def parse_args(): p.add_argument( "--clean", action="store_true", - help="Remove build dir before rebuild (default: False)", + help="Remove build dir before rebuild", ) p.add_argument( "--skip-editable", diff --git a/scripts/gen_coverage.py b/scripts/gen_coverage.py index 17c438eed4..7b203b924f 100644 --- a/scripts/gen_coverage.py +++ b/scripts/gen_coverage.py @@ -28,70 +28,34 @@ build_extension, clean_build_dir, err, + get_output, install_editable, make_cmake_args, + resolve_compilers, run, warn, ) -def find_bin_llvm(compiler_name): - icx_path = subprocess.check_output(["which", compiler_name]) - bin_dir = os.path.dirname(icx_path) - compiler_dir = os.path.join(bin_dir.decode("utf-8"), "compiler") +def find_bin_llvm(compiler): + if os.path.isabs(compiler): + bin_dir = os.path.dirname(compiler) + else: + compiler_path = get_output(["which", compiler]) + if not compiler_path: + raise RuntimeError(f"Compiler {compiler} not found in PATH") + bin_dir = os.path.dirname(compiler_path) + compiler_dir = os.path.join(bin_dir, "compiler") if os.path.exists(compiler_dir): bin_llvm = compiler_dir else: bin_dir = os.path.dirname(bin_dir) - bin_llvm = os.path.join(bin_dir.decode("utf-8"), "bin-llvm") - assert os.path.exists(bin_llvm) + bin_llvm = os.path.join(bin_dir, "bin-llvm") + if not os.path.exists(bin_llvm): + raise RuntimeError(f"--bin-llvm value {bin_llvm} not found") return bin_llvm -def resolve_compilers( - oneapi: bool, - c_compiler: str, - cxx_compiler: str, - compiler_root: str, - bin_llvm: str = None, -): - is_linux = "linux" in sys.platform - - if oneapi or ( - c_compiler is None - and cxx_compiler is None - and compiler_root is None - and bin_llvm is None - ): - return "icx", ("icpx" if is_linux else "icx"), find_bin_llvm("icx") - - if not compiler_root or not os.path.exists(compiler_root): - raise RuntimeError( - "--compiler-root option must be set when using non-default DPC++ " - "layout" - ) - - # default values - if c_compiler is None: - c_compiler = "icx" - if cxx_compiler is None: - cxx_compiler = "icpx" if is_linux else "icx" - if bin_llvm is None: - bin_llvm = find_bin_llvm(c_compiler) - - for name, opt_name in ( - (c_compiler, "--c-compiler"), - (cxx_compiler, "--cxx-compiler"), - (bin_llvm, "--bin-llvm"), - ): - path = ( - name if os.path.exists(name) else os.path.join(compiler_root, name) - ) - if not os.path.exists(path): - raise RuntimeError(f"{opt_name} value {name} not found") - return c_compiler, cxx_compiler, bin_llvm - - def parse_args(): p = argparse.ArgumentParser(description="Build dpctl and generate coverage") @@ -177,13 +141,13 @@ def main(): args = parse_args() setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - c_compiler, cxx_compiler, bin_llvm = resolve_compilers( + c_compiler, cxx_compiler = resolve_compilers( args.oneapi, args.c_compiler, args.cxx_compiler, args.compiler_root, - args.bin_llvm, ) + bin_llvm = find_bin_llvm(c_compiler) if args.clean: clean_build_dir(setup_dir) @@ -246,6 +210,12 @@ def main(): .decode("utf-8") .strip("\n") ) + + cmake_build_dir = get_output( + ["find", "_skbuild", "-name", "cmake-build"], + cwd=setup_dir, + ) + print(f"[gen_coverage] Found CMake build dir: {cmake_build_dir}") run( From 35f91ea02b7fdb8b17dfb684759662313ec0e665 Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 28 Oct 2025 14:54:55 -0700 Subject: [PATCH 15/17] update gen_docs script --- scripts/gen_docs.py | 295 +++++++++++++++++++++++--------------------- 1 file changed, 154 insertions(+), 141 deletions(-) diff --git a/scripts/gen_docs.py b/scripts/gen_docs.py index b5cf1b0602..610b6ea325 100644 --- a/scripts/gen_docs.py +++ b/scripts/gen_docs.py @@ -14,175 +14,188 @@ # See the License for the specific language governing permissions and # limitations under the License. +import argparse import os import subprocess import sys +# add scripts dir to Python path so we can import _build_helper +sys.path.insert(0, os.path.abspath("scripts")) -def run( - use_oneapi=True, - c_compiler=None, - cxx_compiler=None, - level_zero=True, - compiler_root=None, - bin_llvm=None, - doxyrest_dir=None, - verbose=False, - cmake_opts="", -): - IS_LIN = False - - if "linux" in sys.platform: - IS_LIN = True - elif sys.platform in ["win32", "cygwin"]: - pass - else: - assert False, sys.platform + " not supported" - - if not IS_LIN: - raise RuntimeError( - "This scripts only supports coverage collection on Linux" - ) - setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - cmake_args = [ - sys.executable, - "setup.py", - "develop", - "--build-type=Release", - "--generator=Ninja", - "--", - "-DCMAKE_C_COMPILER:PATH=" + c_compiler, - "-DCMAKE_CXX_COMPILER:PATH=" + cxx_compiler, - "-DDPCTL_ENABLE_L0_PROGRAM_CREATION=" + ("ON" if level_zero else "OFF"), - "-DDPCTL_GENERATE_DOCS=ON", - ] - - if verbose: - cmake_args.append("-DCMAKE_VERBOSE_MAKEFILE=ON") - - if doxyrest_dir: - cmake_args.append("-DDPCTL_ENABLE_DOXYREST=ON") - cmake_args.append("-DDoxyrest_DIR=" + doxyrest_dir) - - if cmake_opts: - cmake_args += cmake_opts.split() - - env = dict() - if bin_llvm: - env = { - "PATH": ":".join((os.environ.get("PATH", ""), bin_llvm)), - } - env.update({k: v for k, v in os.environ.items() if k != "PATH"}) - # Install dpctl package - subprocess.check_call(cmake_args, shell=False, cwd=setup_dir, env=env) - # Get the path for the build directory - build_dir = ( - subprocess.check_output( - ["find", "_skbuild", "-name", "cmake-build"], - cwd=setup_dir, - ) - .decode("utf-8") - .strip("\n") - ) - # Generate docs - subprocess.check_call( - ["cmake", "--build", ".", "--target", "Sphinx"], cwd=build_dir - ) - generated_doc_dir = ( - subprocess.check_output( - ["find", "_skbuild", "-name", "index.html"], cwd=setup_dir - ) - .decode("utf-8") - .strip("\n") - ) - print("Generated documentation placed under ", generated_doc_dir) +from _build_helper import ( # noqa: E402 + build_extension, + clean_build_dir, + err, + get_output, + install_editable, + make_cmake_args, + resolve_compilers, + run, + warn, +) -if __name__ == "__main__": - import argparse +def parse_args(): + p = argparse.ArgumentParser(description="Build dpctl and generate coverage") - parser = argparse.ArgumentParser( - description="Driver to build dpctl and generate coverage" + p.add_argument( + "--c-compiler", default=None, help="Path or name of C compiler" + ) + p.add_argument( + "--cxx-compiler", default=None, help="Path or name of C++ compiler" ) - driver = parser.add_argument_group(title="Coverage driver arguments") - driver.add_argument("--c-compiler", help="Name of C compiler", default=None) - driver.add_argument( - "--cxx-compiler", help="Name of C++ compiler", default=None + p.add_argument( + "--compiler-root", + type=str, + default=None, + help="Path to compiler installation root", ) - driver.add_argument( - "--not-oneapi", - help="Is one-API installation", + p.add_argument( + "--oneapi", dest="oneapi", - action="store_false", + action="store_true", + help="Use default oneAPI compiler layout", ) - driver.add_argument( - "--compiler-root", type=str, help="Path to compiler home directory" + + p.add_argument( + "--verbose", + dest="verbose", + action="store_true", + help="Enable verbose makefile output", ) - driver.add_argument( + + p.add_argument( "--no-level-zero", - help="Enable Level Zero support", - dest="level_zero", - action="store_false", + dest="no_level_zero", + action="store_true", + default=False, + help="Disable Level Zero backend (deprecated: use --target-level-zero " + "OFF)", + ) + p.add_argument( + "--target-level-zero", + action="store_true", + help="Enable Level Zero backend explicitly", ) - driver.add_argument( - "--bin-llvm", help="Path to folder where llvm-cov can be found" + + p.add_argument( + "--generator", type=str, default="Ninja", help="CMake generator" + ) + p.add_argument( + "--cmake-executable", + type=str, + default=None, + help="Path to CMake executable used by build", + ) + + p.add_argument( + "--cmake-opts", + type=str, + default="", + help="Additional options to pass directly to CMake", ) - driver.add_argument( + + p.add_argument( "--doxyrest-root", help=( "Path to Doxyrest installation to use to generate Sphinx docs" + "for libsyclinterface" ), ) - driver.add_argument( - "--verbose", - help="Build using vebose makefile mode", - dest="verbose", + + p.add_argument( + "--clean", action="store_true", + help="Remove build dir before rebuild (default: False)", ) - driver.add_argument( - "--cmake-opts", - help="Options to pass through to cmake", - dest="cmake_opts", - default="", - type=str, + + return p.parse_args() + + +def main(): + is_linux = "linux" in sys.platform + if not is_linux: + err(f"{sys.platform} not supported", "gen_docs") + args = parse_args() + setup_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + c_compiler, cxx_compiler = resolve_compilers( + args.oneapi, + args.c_compiler, + args.cxx_compiler, + args.compiler_root, ) - args = parser.parse_args() + if args.clean: + clean_build_dir(setup_dir) - if args.oneapi: - args.c_compiler = "icx" - args.cxx_compiler = "icpx" - args.compiler_root = None - icx_path = subprocess.check_output(["which", "icx"]) - bin_dir = os.path.dirname(icx_path) - args.bin_llvm = os.path.join(bin_dir.decode("utf-8"), "compiler") + if args.no_level_zero and args.target_level_zero: + err( + "Cannot combine --no-level-zero and --target-level-zero", + "gen_coverage", + ) + + # Level Zero state (on unless explicitly disabled) + if args.no_level_zero: + level_zero_enabled = False + elif args.target_level_zero: + level_zero_enabled = True else: - args_to_validate = [ - "c_compiler", - "cxx_compiler", - "compiler_root", - "bin_llvm", - ] - for p in args_to_validate: - arg = getattr(args, p, None) - if not isinstance(arg, str): - opt_name = p.replace("_", "-") - raise RuntimeError( - f"Option {opt_name} must be provided is " - "using non-default DPC++ layout" - ) - if not os.path.exists(arg): - raise RuntimeError(f"Path {arg} must exist") + level_zero_enabled = True - run( - use_oneapi=args.oneapi, - c_compiler=args.c_compiler, - cxx_compiler=args.cxx_compiler, - level_zero=args.level_zero, - compiler_root=args.compiler_root, - bin_llvm=args.bin_llvm, - doxyrest_dir=args.doxyrest_root, + cmake_args = make_cmake_args( + c_compiler=c_compiler, + cxx_compiler=cxx_compiler, + level_zero=level_zero_enabled, verbose=args.verbose, - cmake_opts=args.cmake_opts, ) + + cmake_args += " -DDPCTL_GENERATE_DOCS=ON" + + if args.doxyrest_root: + cmake_args += " -DDPCTL_ENABLE_DOXYREST=ON" + cmake_args += f" -DDoxyrest_DIR={args.doxyrest_root}" + + env = os.environ.copy() + + if "CMAKE_ARGS" in env and env["CMAKE_ARGS"].strip(): + warn("Ignoring pre-existing CMAKE_ARGS in environment", "gen_docs") + del env["CMAKE_ARGS"] + + env["CMAKE_ARGS"] = cmake_args + + print(f"[gen_docs] Using CMake args:\n {env['CMAKE_ARGS']}") + + build_extension( + setup_dir, + env, + cmake_executable=args.cmake_executable, + generator=args.generator, + build_type="Release", + ) + install_editable(setup_dir, env) + cmake_build_dir = get_output( + ["find", "_skbuild", "-name", "cmake-build"], cwd=setup_dir + ) + + print(f"[gen_docs] Found CMake build dir: {cmake_build_dir}") + + run( + ["cmake", "--build", ".", "--target", "Sphinx"], + cwd=cmake_build_dir, + ) + + generated_doc_dir = ( + subprocess.check_output( + ["find", "_skbuild", "-name", "index.html"], cwd=setup_dir + ) + .decode("utf-8") + .strip("\n") + ) + print("Generated documentation placed under ", generated_doc_dir) + + print("[gen_docs] Done") + + +if __name__ == "__main__": + main() From a113d017e9e995fb22107cc135f3b2cfc11b95ea Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 28 Oct 2025 16:03:25 -0700 Subject: [PATCH 16/17] try using pip install -e in place of setup.py develop in CI --- .github/workflows/generate-docs.yml | 3 ++- .github/workflows/os-llvm-sycl-build.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 3e61460eb0..030ee64303 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -75,7 +75,7 @@ jobs: source /opt/intel/oneapi/setvars.sh wget https://github.com/vovkos/doxyrest/releases/download/doxyrest-2.1.2/doxyrest-2.1.2-linux-amd64.tar.xz tar xf doxyrest-2.1.2-linux-amd64.tar.xz - python setup.py develop -G Ninja --build-type=Release \ + python setup.py build_ext --inplace --generator=Ninja --build-type=Release \ -- \ -DCMAKE_C_COMPILER:PATH=$(which icx) \ -DCMAKE_CXX_COMPILER:PATH=$(which icpx) \ @@ -83,6 +83,7 @@ jobs: -DDPCTL_ENABLE_DOXYREST=ON \ -DDoxyrest_DIR=`pwd`/doxyrest-2.1.2-linux-amd64 \ -DCMAKE_VERBOSE_MAKEFILE=ON + python -m pip install -e . --no-build-isolation python -c "import dpctl; print(dpctl.__version__)" || exit 1 pushd "$(find _skbuild -name cmake-build)" || exit 1 cmake --build . --target Sphinx || exit 1 diff --git a/.github/workflows/os-llvm-sycl-build.yml b/.github/workflows/os-llvm-sycl-build.yml index 581b543967..5e171354e3 100644 --- a/.github/workflows/os-llvm-sycl-build.yml +++ b/.github/workflows/os-llvm-sycl-build.yml @@ -146,7 +146,8 @@ jobs: shell: bash -l {0} run: | source set_allvars.sh - CC=clang CXX=clang++ python setup.py develop -G Ninja + CC=clang CXX=clang++ python setup.py build_ext --inplace --generator=Ninja + python -m pip install -e . --no-build-isolation - name: Run lsplatforms shell: bash -l {0} From 5b395e7bd4609c69226063bc11482aaff1e20e0f Mon Sep 17 00:00:00 2001 From: Nikita Grigorian Date: Tue, 28 Oct 2025 16:57:06 -0700 Subject: [PATCH 17/17] use python setup.py build_ext in Cython extension building --- .github/workflows/conda-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml index 6ad93f5b0f..a17c393c60 100644 --- a/.github/workflows/conda-package.yml +++ b/.github/workflows/conda-package.yml @@ -633,7 +633,7 @@ jobs: do pushd $d conda activate --stack ${{ env.BUILD_ENV_NAME }} - CC=icx CXX=icpx python setup.py develop -G Ninja || exit 1 + CC=icx CXX=icpx python setup.py build_ext --inplace -G Ninja || exit 1 conda deactivate python -m pytest tests || exit 1 popd