diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c6cf6d37..c7116b9a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ All pull request authors must have a Contributor License Agreement (CLA) on-file with us. Please sign the Contributor License Agreements for Cloud Foundry ([Individual or Corporate](https://www.cloudfoundry.org/community/cla/)) via the EasyCLA application when you submit your first Pull Request. -When sending signed CLA please provide your github username in case of individual CLA or the list of github usernames that can make pull requests on behalf of your organization. +When sending signed CLA please provide your Github username in case of individual CLA or the list of Github usernames that can make pull requests on behalf of your organization. If you are confident that you're covered under a Corporate CLA, please make sure you've publicized your membership in the appropriate Github Org, per these instructions. diff --git a/README.md b/README.md index 897a55ab..eda068e0 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,20 @@ This buildpack supports running Django and Flask apps. Official buildpack documentation can be found at [python buildpack docs](http://docs.cloudfoundry.org/buildpacks/python/index.html). +## Adding new dependencies + +If you want to add a new dependency to the buildpack, please add it to the [config.yml](https://github.com/cloudfoundry/buildpacks-ci/blob/5a63d13df09f83d5dff7c71d0a12c3e2dc798d39/pipelines/dependency-builds/config.yml#L272) file. For example, if you want to add a new version of Python, add an entry like the following: + +```yaml +python: + lines: + - line: 3.14.X + deprecation_date: 2030-10-07 + link: https://peps.python.org/pep-0745/ +``` + +The new dependency will be automatically added to the buildpack [manifest.yml](manifest.yml) file. + ### Building the Buildpack To build this buildpack, run the following commands from the buildpack's directory: @@ -24,7 +38,7 @@ To build this buildpack, run the following commands from the buildpack's directo 1. Install buildpack-packager ```bash - go install github.com/cloudfoundry/libbuildpack/packager/buildpack-packager + go install github.com/cloudfoundry/libbuildpack/packager/buildpack-packager@latest ``` 1. Build the buildpack diff --git a/src/python/supply/supply.go b/src/python/supply/supply.go index b36a48fd..5dcdd744 100644 --- a/src/python/supply/supply.go +++ b/src/python/supply/supply.go @@ -743,19 +743,74 @@ func (s *Supplier) RunPipVendored() error { // dependencies - wheel and setuptools. These are packaged by the dependency // pipeline within the "pip" dependency. func (s *Supplier) InstallCommonBuildDependencies() error { - var commonDeps = []string{"wheel", "setuptools"} + // wheel and setuptools are packaged as pip-installable sdists inside the + // "pip" dependency tarball. flit-core is a separate dependency whose + // tarball contains the raw Python source tree (not an sdist/wheel for pip). + // + // Bootstrap strategy: + // 1. Extract the pip tarball → /tmp/common_build_deps (wheel/setuptools sdists land here) + // 2. Extract the flit-core tarball → /tmp/common_build_deps + // → /tmp/common_build_deps/flit_core/ (source) + pyproject.toml + // 3. Set PYTHONPATH=/tmp/common_build_deps so flit_core is importable. + // 4. pip install /tmp/common_build_deps --no-build-isolation + // → builds flit_core wheel using itself from PYTHONPATH and installs it. + // 5. pip install wheel setuptools --no-index --no-build-isolation --find-links=tempPath + // → flit_core is now a real installed package, so wheel's build succeeds. tempPath := filepath.Join("/tmp", "common_build_deps") if err := s.Installer.InstallOnlyVersion("pip", tempPath); err != nil { return err } + if err := s.Installer.InstallOnlyVersion("flit-core", tempPath); err != nil { + return err + } + + // Step 3: make flit_core source importable. + prevPythonPath := os.Getenv("PYTHONPATH") + newPythonPath := tempPath + if prevPythonPath != "" { + newPythonPath = tempPath + string(os.PathListSeparator) + prevPythonPath + } + os.Setenv("PYTHONPATH", newPythonPath) + defer os.Setenv("PYTHONPATH", prevPythonPath) - for _, dep := range commonDeps { + // Step 4: install flit_core from its source directory via bootstrap. + s.Log.Info("Installing build-time dependency flit-core (bootstrap)") + if err := s.runPipInstall(tempPath, "--no-build-isolation"); err != nil { + return fmt.Errorf("could not bootstrap-install flit-core: %v", err) + } + + // Step 5: install wheel and setuptools (now flit_core is installed). + for _, dep := range []string{"wheel", "setuptools"} { s.Log.Info("Installing build-time dependency %s", dep) - args := []string{dep, "--no-index", "--upgrade-strategy=only-if-needed", fmt.Sprintf("--find-links=%s", tempPath)} + args := []string{dep, "--no-index", "--no-build-isolation", "--upgrade-strategy=only-if-needed", fmt.Sprintf("--find-links=%s", tempPath)} if err := s.runPipInstall(args...); err != nil { return fmt.Errorf("could not install build-time dependency %s: %v", dep, err) } } + + // Step 6: install poetry-core. + // poetry-core is bundled in the buildpack under vendor_bundled/poetry-core_2.1.3.tgz. + // Its pyproject.toml declares backend-path = ["src"] and requires = [], + // so it self-bootstraps with --no-build-isolation (no build deps required). + bpDir, err := libbuildpack.GetBuildpackDir() + if err != nil { + return fmt.Errorf("could not determine buildpack dir for poetry-core bootstrap: %v", err) + } + poetryCoreTar := filepath.Join(bpDir, "vendor_bundled", "poetry-core_2.1.3.tgz") + poetryCoreSrc := filepath.Join("/tmp", "poetry_core_src") + if err := os.MkdirAll(poetryCoreSrc, 0755); err != nil { + return fmt.Errorf("could not create poetry-core src dir: %v", err) + } + s.Log.Info("Extracting bundled poetry-core from %s", poetryCoreTar) + if err := s.Command.Execute("/", indentWriter(os.Stdout), indentWriter(os.Stderr), + "tar", "xzf", poetryCoreTar, "-C", poetryCoreSrc); err != nil { + return fmt.Errorf("could not extract poetry-core tarball: %v", err) + } + s.Log.Info("Installing build-time dependency poetry-core (bootstrap)") + if err := s.runPipInstall(poetryCoreSrc, "--no-build-isolation"); err != nil { + return fmt.Errorf("could not bootstrap-install poetry-core: %v", err) + } + return nil } diff --git a/src/python/supply/supply_test.go b/src/python/supply/supply_test.go index b7947dc3..cab17d38 100644 --- a/src/python/supply/supply_test.go +++ b/src/python/supply/supply_test.go @@ -632,26 +632,71 @@ MarkupSafe==2.0.1 }) Describe("InstallCommonBuildDependencies", func() { + var bpDir string + + BeforeEach(func() { + bpDir, err = os.MkdirTemp("", "python-buildpack.bp.") + Expect(err).To(BeNil()) + DeferCleanup(os.RemoveAll, bpDir) + // Create the vendor_bundled directory with a dummy poetry-core tarball + Expect(os.MkdirAll(filepath.Join(bpDir, "vendor_bundled"), 0755)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(bpDir, "vendor_bundled", "poetry-core_2.1.3.tgz"), []byte("dummy"), 0644)).To(Succeed()) + // GetBuildpackDir() reads BUILDPACK_DIR env var if set + os.Setenv("BUILDPACK_DIR", bpDir) + DeferCleanup(os.Unsetenv, "BUILDPACK_DIR") + }) + Context("successful installation", func() { - It("runs command to install wheel and setuptools", func() { + It("bootstraps flit-core, wheel, setuptools and poetry-core", func() { + // Step 1+2: install pip and flit-core dependency tarballs mockInstaller.EXPECT().InstallOnlyVersion("pip", "/tmp/common_build_deps") - mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "wheel", "--no-index", "--upgrade-strategy=only-if-needed", "--find-links=/tmp/common_build_deps") - mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "setuptools", "--no-index", "--upgrade-strategy=only-if-needed", "--find-links=/tmp/common_build_deps") + mockInstaller.EXPECT().InstallOnlyVersion("flit-core", "/tmp/common_build_deps") + // Step 4: bootstrap flit_core from source + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "/tmp/common_build_deps", "--no-build-isolation") + // Step 5: install wheel and setuptools + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "wheel", "--no-index", "--no-build-isolation", "--upgrade-strategy=only-if-needed", "--find-links=/tmp/common_build_deps") + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "setuptools", "--no-index", "--no-build-isolation", "--upgrade-strategy=only-if-needed", "--find-links=/tmp/common_build_deps") + // Step 6: extract and bootstrap poetry-core + poetryCoreTar := filepath.Join(bpDir, "vendor_bundled", "poetry-core_2.1.3.tgz") + mockCommand.EXPECT().Execute("/", gomock.Any(), gomock.Any(), "tar", "xzf", poetryCoreTar, "-C", "/tmp/poetry_core_src") + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "/tmp/poetry_core_src", "--no-build-isolation") Expect(supplier.InstallCommonBuildDependencies()).To(Succeed()) }) }) - Context("installation fails", func() { - BeforeEach(func() { - mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "wheel", "--no-index", "--upgrade-strategy=only-if-needed", "--find-links=/tmp/common_build_deps").Return(fmt.Errorf("some-pip-error")) + Context("flit-core bootstrap fails", func() { + It("returns a useful error message", func() { + mockInstaller.EXPECT().InstallOnlyVersion("pip", "/tmp/common_build_deps") + mockInstaller.EXPECT().InstallOnlyVersion("flit-core", "/tmp/common_build_deps") + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "/tmp/common_build_deps", "--no-build-isolation").Return(fmt.Errorf("bootstrap-error")) + Expect(supplier.InstallCommonBuildDependencies()).To(MatchError("could not bootstrap-install flit-core: bootstrap-error")) }) + }) + Context("wheel installation fails", func() { It("returns a useful error message", func() { - mockInstaller.EXPECT().InstallOnlyVersion(gomock.Any(), gomock.Any()).Times(1) + mockInstaller.EXPECT().InstallOnlyVersion("pip", "/tmp/common_build_deps") + mockInstaller.EXPECT().InstallOnlyVersion("flit-core", "/tmp/common_build_deps") + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "/tmp/common_build_deps", "--no-build-isolation") + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "wheel", "--no-index", "--no-build-isolation", "--upgrade-strategy=only-if-needed", "--find-links=/tmp/common_build_deps").Return(fmt.Errorf("some-pip-error")) Expect(supplier.InstallCommonBuildDependencies()).To(MatchError("could not install build-time dependency wheel: some-pip-error")) }) }) + + Context("poetry-core bootstrap fails", func() { + It("returns a useful error message", func() { + mockInstaller.EXPECT().InstallOnlyVersion("pip", "/tmp/common_build_deps") + mockInstaller.EXPECT().InstallOnlyVersion("flit-core", "/tmp/common_build_deps") + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "/tmp/common_build_deps", "--no-build-isolation") + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "wheel", "--no-index", "--no-build-isolation", "--upgrade-strategy=only-if-needed", "--find-links=/tmp/common_build_deps") + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "setuptools", "--no-index", "--no-build-isolation", "--upgrade-strategy=only-if-needed", "--find-links=/tmp/common_build_deps") + poetryCoreTar := filepath.Join(bpDir, "vendor_bundled", "poetry-core_2.1.3.tgz") + mockCommand.EXPECT().Execute("/", gomock.Any(), gomock.Any(), "tar", "xzf", poetryCoreTar, "-C", "/tmp/poetry_core_src") + mockCommand.EXPECT().Execute(buildDir, gomock.Any(), gomock.Any(), "python", "-m", "pip", "install", "/tmp/poetry_core_src", "--no-build-isolation").Return(fmt.Errorf("poetry-error")) + Expect(supplier.InstallCommonBuildDependencies()).To(MatchError("could not bootstrap-install poetry-core: poetry-error")) + }) + }) }) Describe("CreateDefaultEnv", func() {