From c4477f1137ac1be9d313409c43da7e5db678f77a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 16 Apr 2026 11:25:31 +0200 Subject: [PATCH 1/2] Use a dataclass for "branches" --- master/custom/__init__.py | 4 -- master/custom/branches.py | 134 +++++++++++++++++++++++++++++++++++++ master/custom/factories.py | 28 ++++---- master/custom/workers.py | 1 - master/master.cfg | 72 +++++++++----------- 5 files changed, 180 insertions(+), 59 deletions(-) create mode 100644 master/custom/branches.py diff --git a/master/custom/__init__.py b/master/custom/__init__.py index c827f5ae..8456466f 100644 --- a/master/custom/__init__.py +++ b/master/custom/__init__.py @@ -1,5 +1 @@ -MAIN_BRANCH_VERSION = "3.15" -# The Git branch is called "main", but we give it a different name in buildbot. -# See git_branches in master/master.cfg. -MAIN_BRANCH_NAME = "3.x" JUNIT_FILENAME = "test-results.xml" diff --git a/master/custom/branches.py b/master/custom/branches.py new file mode 100644 index 00000000..5d670ddc --- /dev/null +++ b/master/custom/branches.py @@ -0,0 +1,134 @@ +"""All the info about Buildbot branches + +We treat the main branch specially, and we use a pseudo-branch for the +Pull Request buildbots. +In older branches some config needs to be nudged -- for example, +free-threading builds only make sense in 3.13+. + +Complex enough to wrap up in a dataclass: BranchInfo. + + +Run this as a CLI command to print the info out: + + python master/custom/branches.py + +""" + +import dataclasses +from functools import total_ordering +from typing import Any + +# Buildbot configuration first; see below for the BranchInfo class. + +def generate_branches(): + yield BranchInfo( + 'main', + version_tuple=(3, 15), + git_ref='main', + is_main=True, + builddir_name='main', + builder_tag='main', + sort_key=-9999, + ) + yield _maintenance_branch(3, 14) + yield _maintenance_branch(3, 13) + yield _maintenance_branch(3, 12) + yield _maintenance_branch(3, 11) + yield _maintenance_branch(3, 10) + yield BranchInfo( + 'PR', + version_tuple=None, + git_ref=None, + is_pr=True, + builddir_name='pull_request', + builder_tag='PullRequest', + sort_key=0, + ) + + +def _maintenance_branch(major, minor, **kwargs): + version_tuple = (major, minor) + version_str = f'{major}.{minor}' + result = BranchInfo( + name=version_str, + builder_tag=version_str, + version_tuple=version_tuple, + git_ref=version_str, + builddir_name=version_str, + sort_key=-minor, + ) + + if version_tuple < (3, 11): + # Before 3.11, test_asyncio wasn't split out, and refleaks tests + # need more time. + result.monolithic_test_asyncio = True + + if version_tuple < (3, 11): + # WASM wasn't a supported platform until 3.11. + result.wasm_tier = None + elif version_tuple < (3, 13): + # Tier 3 support is 3.11 & 3.12. + result.wasm_tier = 3 + + if version_tuple < (3, 13): + # Free-threaded builds are available since 3.13 + result.gil_only = True + + return result + + +@total_ordering +@dataclasses.dataclass +class BranchInfo: + name: str + version_tuple: tuple[int, int] | None + git_ref: str | None + builddir_name: str + builder_tag: str + + sort_key: Any + + is_main: bool = False + is_pr: bool = False + + # Branch features. + # Defaults are for main (and PR), overrides are in _maintenance_branch. + gil_only: bool = False + monolithic_test_asyncio: bool = False + wasm_tier: int | None = 2 + + def __str__(self): + return self.name + + def __eq__(self, other): + try: + other_key = other.sort_key + except AttributeError: + return NotImplemented + return self.sort_key == other.sort_key + + def __lt__(self, other): + try: + other_key = other.sort_key + except AttributeError: + return NotImplemented + return self.sort_key < other.sort_key + + +BRANCHES = list(generate_branches()) +PR_BRANCH = BRANCHES[-1] + +# Verify that we've defined these in sort order +assert BRANCHES == sorted(BRANCHES) + +if __name__ == "__main__": + # Print a table to the terminal + cols = [[f.name + ':' for f in dataclasses.fields(BranchInfo)]] + for branch in BRANCHES: + cols.append([repr(val) for val in dataclasses.astuple(branch)]) + column_sizes = [max(len(val) for val in col) for col in cols] + column_sizes[-2] += 2 # PR is special, offset it a bit + for row in zip(*cols): + for size, val in zip(column_sizes, row): + print(val.ljust(size), end=' ') + print() diff --git a/master/custom/factories.py b/master/custom/factories.py index 0b181c12..6f382cd0 100644 --- a/master/custom/factories.py +++ b/master/custom/factories.py @@ -9,8 +9,7 @@ from buildbot.plugins import util -from . import (MAIN_BRANCH_VERSION, MAIN_BRANCH_NAME, - JUNIT_FILENAME) +from . import JUNIT_FILENAME from .steps import ( Test, Clean, @@ -56,6 +55,8 @@ def get_j_opts(worker, default=None): class BaseBuild(factory.BuildFactory): factory_tags = [] test_timeout = TEST_TIMEOUT + buildersuffix = "" + tags = () def __init__(self, source, *, extra_tags=[], **kwargs): super().__init__([source]) @@ -97,7 +98,7 @@ def setup(self, branch, worker, test_with_PTY=False, **kwargs): # In 3.10, test_asyncio wasn't split out, and refleaks tests # need more time. - if branch == "3.10" and has_option("-R", self.testFlags): + if branch.monolithic_test_asyncio and has_option("-R", self.testFlags): self.test_timeout *= 2 if self.build_out_of_tree: @@ -161,7 +162,7 @@ def setup(self, branch, worker, test_with_PTY=False, **kwargs): env=self.test_environ, **oot_kwargs )) - if branch not in ("3",) and not has_option("-R", self.testFlags): + if not branch.is_pr and not has_option("-R", self.testFlags): filename = JUNIT_FILENAME if self.build_out_of_tree: filename = os.path.join(out_of_tree_dir, filename) @@ -214,11 +215,12 @@ class UnixInstalledBuild(BaseBuild): factory_tags = ["installed"] def setup(self, branch, worker, test_with_PTY=False, **kwargs): - if branch == MAIN_BRANCH_NAME: - branch = MAIN_BRANCH_VERSION - elif branch == "custom": - branch = "3" - installed_python = f"./target/bin/python{branch}" + if branch.version_tuple: + major, minor = branch.version_tuple + executable_name = f'python{major}.{minor}' + else: + executable_name = f'python3' + installed_python = f"./target/bin/{executable_name}" self.addStep( Configure( command=["./configure", "--prefix", "$(PWD)/target"] @@ -633,7 +635,7 @@ def setup(self, branch, worker, **kwargs): command=test_command, timeout=step_timeout(self.test_timeout), )) - if branch not in ("3",) and not has_option("-R", self.testFlags): + if not branch.is_pr and not has_option("-R", self.testFlags): self.addStep(UploadTestResults(branch)) self.addStep(Clean(command=clean_command)) @@ -856,7 +858,7 @@ def setup(self, branch, worker, test_with_PTY=False, **kwargs): env=self.test_environ, workdir=oot_host_path, )) - if branch not in ("3",) and not has_option("-R", self.testFlags): + if not branch.is_pr and not has_option("-R", self.testFlags): filename = os.path.join(oot_host_path, JUNIT_FILENAME) self.addStep(UploadTestResults(branch, filename=filename)) self.addStep( @@ -990,7 +992,7 @@ def setup(self, branch, worker, test_with_PTY=False, **kwargs): workdir=host_path, ) ) - if branch not in ("3",) and not has_option("-R", self.testFlags): + if not branch.is_pr and not has_option("-R", self.testFlags): filename = os.path.join(host_path, JUNIT_FILENAME) self.addStep(UploadTestResults(branch, filename=filename)) @@ -1240,7 +1242,7 @@ def setup(self, branch, *args, **kwargs): # # The symlink approach will fail for Python 3.13 *PR* builds, because # there's no way to identify the base branch for a PR. - if branch == "3.13": + if branch.name == "3.13": self.py313_setup(branch, *args, **kwargs) else: self.current_setup(branch, *args, **kwargs) diff --git a/master/custom/workers.py b/master/custom/workers.py index c59d4fed..639f246b 100644 --- a/master/custom/workers.py +++ b/master/custom/workers.py @@ -8,7 +8,6 @@ from buildbot.plugins import worker as _worker -from custom.factories import MAIN_BRANCH_NAME from custom.worker_downtime import no_builds_between diff --git a/master/master.cfg b/master/master.cfg index 9c56ed53..64476a21 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -32,7 +32,6 @@ for k in list(sys.modules): if k.split(".")[0] in ["custom"]: sys.modules.pop(k) -from custom import MAIN_BRANCH_NAME # noqa: E402 from custom.auth import set_up_authorization # noqa: E402 from custom.email_formatter import MESSAGE_FORMATTER # noqa: E402 from custom.pr_reporter import GitHubPullRequestReporter # noqa: E402 @@ -51,6 +50,7 @@ from custom.builders import ( # noqa: E402 STABLE, ONLY_MAIN_BRANCH, ) +from custom.branches import BRANCHES, PR_BRANCH def set_up_sentry(): @@ -128,16 +128,9 @@ c["caches"] = { # workers are set up in workers.py c["workers"] = [w.bb_worker for w in WORKERS] -# repo url, buildbot category name, git branch name -git_url = str(settings.git_url) -git_branches = [ - (git_url, MAIN_BRANCH_NAME, "main"), - (git_url, "3.14", "3.14"), - (git_url, "3.13", "3.13"), - (git_url, "3.12", "3.12"), - (git_url, "3.11", "3.11"), - (git_url, "3.10", "3.10"), -] +GIT_URL = str(settings.git_url) + +# git_branches used to be here; moved to branches.py # common Git() and GitHub() keyword arguments GIT_KWDS = { @@ -199,47 +192,45 @@ mail_status_builders = [] # Regular builders -for git_url, branchname, git_branch in git_branches: +for branch in BRANCHES: + if not branch.git_ref: + continue buildernames = [] refleakbuildernames = [] for name, worker, buildfactory, stability, tier in BUILDERS: if any( pattern in name for pattern in ONLY_MAIN_BRANCH - ) and branchname != MAIN_BRANCH_NAME: + ) and not branch.is_main: # Workers known to be broken on older branches: let's focus on # supporting these platforms in the main branch. continue - if worker.not_branches and branchname in worker.not_branches: + if worker.not_branches and branch.name in worker.not_branches: continue - if worker.branches and branchname not in worker.branches: + if worker.branches and branch.name not in worker.branches: continue - buildername = name + " " + branchname - source = Git(repourl=git_url, branch=git_branch, **GIT_KWDS) + buildername = name + " " + branch.name + source = Git(repourl=GIT_URL, branch=branch.git_ref, **GIT_KWDS) f = buildfactory( source, - branch=branchname, + branch=branch, worker=worker, ) - tags = [branchname, stability, *getattr(f, "tags", [])] + tags = [branch.builder_tag, stability, *f.tags] if tier: tags.append(tier) - # Only 3.11+ for WebAssembly builds + # Tiers for WebAssembly builds if "wasm" in tags: - # WASM wasn't a supported platform until 3.11. - if branchname in {"3.10"}: + if branch.wasm_tier is None: continue - # Tier 3 support is 3.11 & 3.12. - elif "nondebug" in tags and branchname not in {"3.11", "3.12"}: + elif "nondebug" in tags and branch.wasm_tier == 3: continue - # Tier 2 support is 3.13+. - elif "nondebug" not in tags and branchname in {"3.11", "3.12"}: + elif "nondebug" not in tags and branch.wasm_tier == 2: continue - # Only 3.13+ for NoGIL builds - if 'nogil' in tags and branchname in {"3.10", "3.11", "3.12"}: + if 'nogil' in tags and branch.gil_only: continue if 'refleak' in tags: @@ -257,7 +248,7 @@ for git_url, branchname, git_branch in git_branches: builder = util.BuilderConfig( name=buildername, workernames=[worker.name], - builddir="%s.%s%s" % (branchname, worker.name, getattr(f, "buildersuffix", "")), + builddir=f"{branch.builddir_name}.{worker.name}{f.buildersuffix}", factory=f, tags=tags, locks=[cpulock.access("counting")], @@ -270,8 +261,8 @@ for git_url, branchname, git_branch in git_branches: c["schedulers"].append( schedulers.SingleBranchScheduler( - name=branchname, - change_filter=util.ChangeFilter(branch=git_branch), + name=branch.name, + change_filter=util.ChangeFilter(branch=branch.git_ref), treeStableTimer=30, # seconds builderNames=buildernames, fileIsImportant=is_important_change, @@ -280,8 +271,8 @@ for git_url, branchname, git_branch in git_branches: if refleakbuildernames: c["schedulers"].append( schedulers.SingleBranchScheduler( - name=branchname + "-refleak", - change_filter=util.ChangeFilter(branch=git_branch), + name=branch.name + "-refleak", + change_filter=util.ChangeFilter(branch=branch.git_ref), # Wait this many seconds for no commits before starting a build # NB: During extremely busy times, this can cause the builders # to never actually fire. The current expectation is that it @@ -300,20 +291,19 @@ stable_pull_request_builders = [] all_pull_request_builders = [] for name, worker, buildfactory, stability, tier in BUILDERS: - buildername = f"{name} PR" + branch = PR_BRANCH + assert PR_BRANCH.is_pr - source = GitHub(repourl=git_url, **GIT_KWDS) + buildername = f"{name} PR" + source = GitHub(repourl=GIT_URL, **GIT_KWDS) f = buildfactory( source, - # Use the same downstream branch names as the "custom" - # builder (check what the factories are doing with this - # parameter for more info). - branch="3", + branch=branch, worker=worker, ) - tags = ["PullRequest", stability, *getattr(f, "tags", [])] + tags = [branch.builder_tag, stability, *f.tags] if tier: tags.append(tier) @@ -330,7 +320,7 @@ for name, worker, buildfactory, stability, tier in BUILDERS: builder = util.BuilderConfig( name=buildername, workernames=[worker.name], - builddir="%s.%s%s" % ("pull_request", worker.name, getattr(f, "buildersuffix", "")), + builddir=f"{branch.builddir_name}.{worker.name}{f.buildersuffix}", factory=f, tags=tags, locks=[cpulock.access("counting")], From ee48c41619ae2c6b032a345a69589c13a411142e Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 16 Apr 2026 13:55:16 +0200 Subject: [PATCH 2/2] Rename git_ref to git_branch --- master/custom/branches.py | 8 ++++---- master/master.cfg | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/master/custom/branches.py b/master/custom/branches.py index 5d670ddc..22bbed3e 100644 --- a/master/custom/branches.py +++ b/master/custom/branches.py @@ -24,7 +24,7 @@ def generate_branches(): yield BranchInfo( 'main', version_tuple=(3, 15), - git_ref='main', + git_branch='main', is_main=True, builddir_name='main', builder_tag='main', @@ -38,7 +38,7 @@ def generate_branches(): yield BranchInfo( 'PR', version_tuple=None, - git_ref=None, + git_branch=None, is_pr=True, builddir_name='pull_request', builder_tag='PullRequest', @@ -53,7 +53,7 @@ def _maintenance_branch(major, minor, **kwargs): name=version_str, builder_tag=version_str, version_tuple=version_tuple, - git_ref=version_str, + git_branch=version_str, builddir_name=version_str, sort_key=-minor, ) @@ -82,7 +82,7 @@ def _maintenance_branch(major, minor, **kwargs): class BranchInfo: name: str version_tuple: tuple[int, int] | None - git_ref: str | None + git_branch: str | None builddir_name: str builder_tag: str diff --git a/master/master.cfg b/master/master.cfg index 64476a21..9451a462 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -193,7 +193,7 @@ mail_status_builders = [] # Regular builders for branch in BRANCHES: - if not branch.git_ref: + if not branch.git_branch: continue buildernames = [] refleakbuildernames = [] @@ -211,7 +211,7 @@ for branch in BRANCHES: continue buildername = name + " " + branch.name - source = Git(repourl=GIT_URL, branch=branch.git_ref, **GIT_KWDS) + source = Git(repourl=GIT_URL, branch=branch.git_branch, **GIT_KWDS) f = buildfactory( source, branch=branch, @@ -262,7 +262,7 @@ for branch in BRANCHES: c["schedulers"].append( schedulers.SingleBranchScheduler( name=branch.name, - change_filter=util.ChangeFilter(branch=branch.git_ref), + change_filter=util.ChangeFilter(branch=branch.git_branch), treeStableTimer=30, # seconds builderNames=buildernames, fileIsImportant=is_important_change, @@ -272,7 +272,7 @@ for branch in BRANCHES: c["schedulers"].append( schedulers.SingleBranchScheduler( name=branch.name + "-refleak", - change_filter=util.ChangeFilter(branch=branch.git_ref), + change_filter=util.ChangeFilter(branch=branch.git_branch), # Wait this many seconds for no commits before starting a build # NB: During extremely busy times, this can cause the builders # to never actually fire. The current expectation is that it