Skip to content

Improve Windows builds #9

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Schamper merged 1 commit into from
Apr 23, 2026
Merged

Improve Windows builds #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 22 additions & 11 deletions cpython-windows/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
convert_to_static_library,
copy_link_to_lib,
hack_source_files,
link_builtin_extensions_into_executables,
remove_from_config_c,
remove_from_extension_modules,
)
Expand Down Expand Up @@ -751,6 +752,8 @@ def hack_project_files(

pythoncore_proj = pcbuild_path / "pythoncore.vcxproj"

converted_builtin_extensions: list[str] = []

for extension, entry in sorted(CONVERT_TO_BUILTIN_EXTENSIONS.items()):
if entry.get("ignore_static") or extension in DISABLED_EXTENSIONS:
log("ignoring extension %s in static builds" % extension)
Expand All @@ -760,6 +763,14 @@ def hack_project_files(

if convert_to_static_library(cpython_source_path, extension, entry, False):
add_to_config_c(cpython_source_path, extension, init_fn)
converted_builtin_extensions.append(extension)

# Link each built-in extension's static library directly into
# python.exe / pythonw.exe to resolve the PyInit_* references in
# PC/config.c.
link_builtin_extensions_into_executables(
cpython_source_path, converted_builtin_extensions
)

# pythoncore.vcxproj produces libpython. Typically pythonXY.dll. We change
# it to produce a static library.
Expand Down Expand Up @@ -835,6 +846,17 @@ def hack_pystandalone_files(source_path: pathlib.Path, python_version: str):
# PYSTANDALONE: add pystandalone module to config.c
add_to_config_c(source_path, "_pystandalone", "PyInit__pystandalone")

# Register _pystandalone in pcbuild.proj's ExtensionModules list so
# MSBuild picks it up via `<Projects Include="@(ExtensionModules->'%(Identity).vcxproj')" />`.
pcbuild_proj_path = source_path / "PCbuild" / "pcbuild.proj"
pcbuild_proj_text = pcbuild_proj_path.read_text(encoding="utf8")
if "_pystandalone" not in pcbuild_proj_text:
static_replace_in_file(
pcbuild_proj_path,
b'<ExtensionModules Include="_asyncio',
b'<ExtensionModules Include="_pystandalone;_asyncio',
)

# PYSTANDALONE: apply patches to Python source
python_major_minor_version = python_version.rsplit(".", 1)[0]
if (
Expand Down Expand Up @@ -1694,17 +1716,6 @@ def build_cpython(
)
hack_source_files(cpython_source_path, python_version=python_version)

# CPython 3.13+ renamed PC/pyconfig.h to PC/pyconfig.h.in and
# generates pyconfig.h at build time via the _UpdatePyconfig target
# in pythoncore.vcxproj. Since we reorder extensions to build
# before pythoncore, pyconfig.h does not exist yet when extensions
# compile. Pre-generate it so the PC/ include-path entry works.
pyconfig_h = cpython_source_path / "PC" / "pyconfig.h"
pyconfig_h_in = cpython_source_path / "PC" / "pyconfig.h.in"
if not pyconfig_h.exists() and pyconfig_h_in.exists():
log("pre-generating PC/pyconfig.h from PC/pyconfig.h.in")
pyconfig_h.write_bytes(pyconfig_h_in.read_bytes())

if pgo:
# PYSTANDALONE: build once to regenerate the frozen zipimport
# In newer Python versions, regenerating frozen modules is done automatically
Expand Down
207 changes: 72 additions & 135 deletions pythonbuild/static.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -227,150 +227,87 @@ def convert_to_static_library(
" </ProjectReference>",
]

# Ensure the extension project doesn't depend on pythoncore: as a built-in
# extension, pythoncore will depend on it.

# This logic is a bit hacky. Ideally we'd parse the file as XML and operate
# in the XML domain. But that is more work. The goal here is to strip the
# <ProjectReference>...</ProjectReference> containing the
# <Project>{pythoncore ID}</Project>. This could leave an item <ItemGroup>.
# That should be fine.
start_line, end_line = None, None
for i, line in enumerate(lines):
if "<Project>{cf7ac3d1-e2df-41d2-bea6-1e2556cdea26}</Project>" in line:
for j in range(i, 0, -1):
if "<ProjectReference" in lines[j]:
start_line = j
break

for j in range(i, len(lines) - 1):
if "</ProjectReference>" in lines[j]:
end_line = j
break

break

if start_line is not None and end_line is not None:
log("stripping pythoncore dependency from %s" % extension)
for line in lines[start_line : end_line + 1]:
log(line)

lines = lines[:start_line] + lines[end_line + 1 :]

# Preserve the natural dependency graph: extensions depend on
# pythoncore. This ensures pythoncore's _UpdatePyconfig target runs
# before any extension compiles, so pyconfig.h exists in pythoncore's
# IntDir and extensions see a consistent, correctly-substituted copy
# via GeneratedPyConfigDir on /I.
#
# pythoncore.lib therefore does not contain extension objs; the final
# executables (python.exe, pythonw.exe) link each extension's .lib
# directly to resolve the PyInit_* symbols referenced by PC/config.c.
# See `link_builtin_extensions_into_executables` below.
with proj_path.open("w", encoding="utf8") as fh:
fh.write("\n".join(lines))

# Tell pythoncore to link against the static .lib.
RE_ADDITIONAL_DEPENDENCIES = re.compile(
"<AdditionalDependencies>([^<]+)</AdditionalDependencies>"
)

pythoncore_path = source_path / "PCbuild" / "pythoncore.vcxproj"
lines = []

with pythoncore_path.open("r", encoding="utf8") as fh:
for line in fh:
line = line.rstrip()

m = RE_ADDITIONAL_DEPENDENCIES.search(line)

if m:
log("changing pythoncore to link against %s.lib" % extension)
# TODO we shouldn't need this with static linking if the
# project is configured to link library dependencies.
# But removing it results in unresolved external symbols
# when linking the python project. There /might/ be a
# visibility issue with the PyMODINIT_FUNC macro.
line = line.replace(
m.group(1), r"$(OutDir)%s.lib;%s" % (extension, m.group(1))
)

lines.append(line)

with pythoncore_path.open("w", encoding="utf8") as fh:
fh.write("\n".join(lines))

# Change pythoncore to depend on the extension project.

# pcbuild.proj is the file that matters for msbuild. And order within
# matters. We remove the extension from the "ExtensionModules" set of
# projects. Then we re-add the project to before "pythoncore."
remove_from_extension_modules(source_path, extension)

pcbuild_proj_path = source_path / "PCbuild" / "pcbuild.proj"

with pcbuild_proj_path.open("r", encoding="utf8") as fh:
data = fh.read()

data = data.replace(
'<Projects Include="pythoncore.vcxproj">',
' <Projects Include="%s.vcxproj" />\n <Projects Include="pythoncore.vcxproj">'
% extension,
)

with pcbuild_proj_path.open("w", encoding="utf8") as fh:
fh.write(data)

# We don't technically need to modify the solution since msbuild doesn't
# use it. But it enables debugging inside Visual Studio, which is
# convenient.
RE_PROJECT = re.compile(
r'Project\("\{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942\}"\) = "([^"]+)", "[^"]+", "{([^\}]+)\}"'
)

pcbuild_sln_path = source_path / "PCbuild" / "pcbuild.sln"
lines = []

extension_id = None
pythoncore_line = None

with pcbuild_sln_path.open("r", encoding="utf8") as fh:
# First pass buffers the file, finds the ID of the extension project,
# and finds where the pythoncore project is defined.
for i, line in enumerate(fh):
line = line.rstrip()

m = RE_PROJECT.search(line)

if m and m.group(1) == extension:
extension_id = m.group(2)

if m and m.group(1) == "pythoncore":
pythoncore_line = i

lines.append(line)

# Not all projects are in the solution(!!!). Since we don't use the
# solution for building, that's fine to ignore.
if not extension_id:
log("failed to find project %s in solution" % extension)
return True

if not pythoncore_line:
log("failed to find pythoncore project in solution")

if extension_id and pythoncore_line:
log("making pythoncore depend on %s" % extension)
def link_builtin_extensions_into_executables(
source_path: pathlib.Path, extensions: list[str]
):
"""Make python.exe / pythonw.exe link against each built-in extension's
static library.

python.vcxproj / pythonw.vcxproj inherit their link settings from
pyproject.props and do not declare an `<AdditionalDependencies>`
element. Inject one just before the closing `</Link>` listing
`$(OutDir)<ext>.lib` for every converted extension so the linker
resolves the `PyInit_*` symbols referenced by PC/config.c.
"""

needs_section = (
not lines[pythoncore_line + 1].lstrip().startswith("ProjectSection")
if not extensions:
return

additional = ";".join("$(OutDir)%s.lib" % ext for ext in extensions)
injected_line = (
" <AdditionalDependencies>"
"%s;%%(AdditionalDependencies)"
"</AdditionalDependencies>"
) % additional

for exe_proj_name in ("python", "pythonw", "_freeze_importlib"):
exe_path = source_path / "PCbuild" / ("%s.vcxproj" % exe_proj_name)
if not exe_path.exists():
log("skipping %s.vcxproj: not present" % exe_proj_name)
continue

with exe_path.open("r", encoding="utf8") as fh:
data = fh.read()

# Detect likely newline style so we can preserve it.
newline = "\r\n" if "\r\n" in data else "\n"

closing = newline + " </Link>"
if closing not in data:
log(
"warning: no </Link> tag found in %s.vcxproj; "
"built-in extensions will not link into %s.exe"
% (exe_proj_name, exe_proj_name)
)
continue

# Insert only once (idempotent).
if "<!-- PYSTANDALONE_BUILTIN_EXT_LIBS -->" in data:
log("skipping %s.vcxproj: already patched" % exe_proj_name)
continue

replacement = (
newline
+ injected_line
+ newline
+ " <!-- PYSTANDALONE_BUILTIN_EXT_LIBS -->"
+ newline
+ " </Link>"
)
offset = 1 if needs_section else 2
data = data.replace(closing, replacement, 1)

lines.insert(
pythoncore_line + offset, "\t\t{%s} = {%s}" % (extension_id, extension_id)
log(
"linking %d built-in extension lib(s) into %s.exe"
% (len(extensions), exe_proj_name)
)

if needs_section:
lines.insert(
pythoncore_line + 1,
"\tProjectSection(ProjectDependencies) = postProject",
)
lines.insert(pythoncore_line + 3, "\tEndProjectSection")

with pcbuild_sln_path.open("w", encoding="utf8") as fh:
fh.write("\n".join(lines))

return True
with exe_path.open("w", encoding="utf8") as fh:
fh.write(data)


def copy_link_to_lib(p: pathlib.Path):
Expand Down
7 changes: 3 additions & 4 deletions src/release.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,6 @@ pub static RELEASE_TRIPLES: Lazy<BTreeMap<&'static str, TripleRelease>> = Lazy::
);

// Windows.
// PYSTANDALONE: we can only build freethreaded on >= 3.14
h.insert(
"i686-pc-windows-msvc",
TripleRelease {
Expand All @@ -118,7 +117,7 @@ pub static RELEASE_TRIPLES: Lazy<BTreeMap<&'static str, TripleRelease>> = Lazy::
freethreaded_install_only_suffix: "freethreaded+pgo",
python_version_requirement: None,
conditional_suffixes: vec![ConditionalSuffixes {
python_version_requirement: VersionSpecifier::from_str(">=3.14").unwrap(),
python_version_requirement: VersionSpecifier::from_str(">=3.13").unwrap(),
suffixes: vec!["freethreaded+pgo"],
}],
},
Expand All @@ -131,7 +130,7 @@ pub static RELEASE_TRIPLES: Lazy<BTreeMap<&'static str, TripleRelease>> = Lazy::
freethreaded_install_only_suffix: "freethreaded+pgo",
python_version_requirement: None,
conditional_suffixes: vec![ConditionalSuffixes {
python_version_requirement: VersionSpecifier::from_str(">=3.14").unwrap(),
python_version_requirement: VersionSpecifier::from_str(">=3.13").unwrap(),
suffixes: vec!["freethreaded+pgo"],
}],
},
Expand All @@ -144,7 +143,7 @@ pub static RELEASE_TRIPLES: Lazy<BTreeMap<&'static str, TripleRelease>> = Lazy::
freethreaded_install_only_suffix: "freethreaded+pgo",
python_version_requirement: Some(VersionSpecifier::from_str(">=3.11").unwrap()),
conditional_suffixes: vec![ConditionalSuffixes {
python_version_requirement: VersionSpecifier::from_str(">=3.14").unwrap(),
python_version_requirement: VersionSpecifier::from_str(">=3.13").unwrap(),
suffixes: vec!["freethreaded+pgo"],
}],
},
Expand Down
Loading