gh-138451: Support custom LLVM installation path (#138452)

Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
Co-authored-by: Steve Dower <steve.dower@microsoft.com>
Co-authored-by: Savannah Ostrowski <savannah@python.org>

Author: 4 people

Misc/NEWS.d/next/Build/2025-09-03-14-55-59.gh-issue-138451.-Qzh2S.rst

@@ -0,0 +1 @@
+Allow for custom LLVM path using ``LLVM_TOOLS_INSTALL_DIR`` during JIT build.

PCbuild/regen.targets

@@ -129,7 +129,7 @@
       <JITArgs Condition="$(Platform) == 'x64'">x86_64-pc-windows-msvc</JITArgs>
       <JITArgs Condition="$(Configuration) == 'Debug'">$(JITArgs) --debug</JITArgs>
     </PropertyGroup>
-    <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs) --output-dir "$(GeneratedJitStencilsDir)" --pyconfig-dir "$(PySourcePath)PC"'/>
+    <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs) --output-dir "$(GeneratedJitStencilsDir)" --pyconfig-dir "$(PySourcePath)PC" --llvm-version="$(LLVM_VERSION)" --llvm-tools-install-dir="$(LLVM_TOOLS_INSTALL_DIR)"'/>
   </Target>
   <Target Name="_CleanJIT" AfterTargets="Clean">
     <Delete Files="@(_JITOutputs)"/>

Tools/jit/README.md

@@ -9,7 +9,12 @@ Python 3.11 or newer is required to build the JIT.
 
 The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon).
 
-LLVM version 21 is the officially supported version. You can modify if needed using the `LLVM_VERSION` env var during configure. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-19`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code.
+LLVM version 21 is the officially supported version. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-21`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code.
+
+You can customize the LLVM configuration using environment variables before running configure:
+
+- LLVM_VERSION: Specify a different LLVM version (default: 21)
+- LLVM_TOOLS_INSTALL_DIR: Point to a specific LLVM installation prefix when multiple installations exist (the tools are expected in `<dir>/bin`)
 
 It's easy to install all of the required tools:
 
@@ -62,7 +67,7 @@ choco install llvm --version=21.1.0
 
 ### Dev Containers
 
-If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no 
+If you are working on CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no
 need to install LLVM as the Fedora 43 base image includes LLVM 21 out of the box.
 
 ## Building

Tools/jit/_llvm.py

@@ -80,7 +80,18 @@ async def _get_brew_llvm_prefix(llvm_version: str, *, echo: bool = False) -> str
 
 
 @_async_cache
-async def _find_tool(tool: str, llvm_version: str, *, echo: bool = False) -> str | None:
+async def _find_tool(
+    tool: str,
+    llvm_version: str,
+    llvm_tools_install_dir: str | None,
+    *,
+    echo: bool = False,
+) -> str | None:
+    # Explicitly defined LLVM installation location
+    if llvm_tools_install_dir:
+        path = os.path.join(llvm_tools_install_dir, "bin", tool)
+        if await _check_tool_version(path, llvm_version, echo=echo):
+            return path
     # Unversioned executables:
     path = tool
     if await _check_tool_version(path, llvm_version, echo=echo):
@@ -114,10 +125,11 @@ async def maybe_run(
     args: typing.Iterable[str],
     echo: bool = False,
     llvm_version: str = _LLVM_VERSION,
+    llvm_tools_install_dir: str | None = None,
 ) -> str | None:
     """Run an LLVM tool if it can be found. Otherwise, return None."""
 
-    path = await _find_tool(tool, llvm_version, echo=echo)
+    path = await _find_tool(tool, llvm_version, llvm_tools_install_dir, echo=echo)
     return path and await _run(path, args, echo=echo)
 
 
@@ -126,10 +138,17 @@ async def run(
     args: typing.Iterable[str],
     echo: bool = False,
     llvm_version: str = _LLVM_VERSION,
+    llvm_tools_install_dir: str | None = None,
 ) -> str:
     """Run an LLVM tool if it can be found. Otherwise, raise RuntimeError."""
 
-    output = await maybe_run(tool, args, echo=echo, llvm_version=llvm_version)
+    output = await maybe_run(
+        tool,
+        args,
+        echo=echo,
+        llvm_version=llvm_version,
+        llvm_tools_install_dir=llvm_tools_install_dir,
+    )
     if output is None:
         raise RuntimeError(f"Can't find {tool}-{llvm_version}!")
     return output

Tools/jit/_targets.py

@@ -53,6 +53,7 @@ class _Target(typing.Generic[_S, _R]):
     cflags: str = ""
     frame_pointers: bool = False
     llvm_version: str = _llvm._LLVM_VERSION
+    llvm_tools_install_dir: str | None = None
     known_symbols: dict[str, int] = dataclasses.field(default_factory=dict)
     pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve()
 
@@ -85,7 +86,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup:
         group = _stencils.StencilGroup()
         args = ["--disassemble", "--reloc", f"{path}"]
         output = await _llvm.maybe_run(
-            "llvm-objdump", args, echo=self.verbose, llvm_version=self.llvm_version
+            "llvm-objdump",
+            args,
+            echo=self.verbose,
+            llvm_version=self.llvm_version,
+            llvm_tools_install_dir=self.llvm_tools_install_dir,
         )
         if output is not None:
             # Make sure that full paths don't leak out (for reproducibility):
@@ -105,7 +110,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup:
             f"{path}",
         ]
         output = await _llvm.run(
-            "llvm-readobj", args, echo=self.verbose, llvm_version=self.llvm_version
+            "llvm-readobj",
+            args,
+            echo=self.verbose,
+            llvm_version=self.llvm_version,
+            llvm_tools_install_dir=self.llvm_tools_install_dir,
         )
         # --elf-output-style=JSON is only *slightly* broken on Mach-O...
         output = output.replace("PrivateExtern\n", "\n")
@@ -184,7 +193,11 @@ async def _compile(
         # Allow user-provided CFLAGS to override any defaults
         args_s += shlex.split(self.cflags)
         await _llvm.run(
-            "clang", args_s, echo=self.verbose, llvm_version=self.llvm_version
+            "clang",
+            args_s,
+            echo=self.verbose,
+            llvm_version=self.llvm_version,
+            llvm_tools_install_dir=self.llvm_tools_install_dir,
         )
         if not is_shim:
             self.optimizer(
@@ -196,7 +209,11 @@ async def _compile(
             ).run()
         args_o = [f"--target={self.triple}", "-c", "-o", f"{o}", f"{s}"]
         await _llvm.run(
-            "clang", args_o, echo=self.verbose, llvm_version=self.llvm_version
+            "clang",
+            args_o,
+            echo=self.verbose,
+            llvm_version=self.llvm_version,
+            llvm_tools_install_dir=self.llvm_tools_install_dir,
         )
         return await self._parse(o)
 

Tools/jit/build.py

@@ -43,6 +43,9 @@
         "--cflags", help="additional flags to pass to the compiler", default=""
     )
     parser.add_argument("--llvm-version", help="LLVM version to use")
+    parser.add_argument(
+        "--llvm-tools-install-dir", help="Installation location of LLVM tools"
+    )
     args = parser.parse_args()
     for target in args.target:
         target.debug = args.debug
@@ -52,6 +55,8 @@
         target.pyconfig_dir = args.pyconfig_dir
         if args.llvm_version:
             target.llvm_version = args.llvm_version
+        if args.llvm_tools_install_dir:
+            target.llvm_tools_install_dir = args.llvm_tools_install_dir
         target.build(
             comment=comment,
             force=args.force,

configure

@@ -2850,7 +2850,7 @@ AS_VAR_IF([jit_flags],
           [],
           [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"])
            AS_VAR_SET([REGEN_JIT_COMMAND],
-                      ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\""])
+                      ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\" --llvm-version=\"$LLVM_VERSION\" --llvm-tools-install-dir=\"$LLVM_TOOLS_INSTALL_DIR\""])
            AS_VAR_IF([Py_DEBUG],
                      [true],
                      [AS_VAR_APPEND([REGEN_JIT_COMMAND], [" --debug"])],