[elbe-devel] [PATCH v3 2/4] pbuilder: adjust pdebuild_build and create_pbuilder
Torben Hohn
torben.hohn at linutronix.de
Wed Apr 29 16:06:23 CEST 2020
On Mon, Apr 27, 2020 at 04:35:42PM +0200, Christian Teklenborg wrote:
> Create a build environment for crossbuilding packages if the 'elbe pbuilder
> --cross create' command is called. Now you can run 'elbe pbuilder --cross build'
> to use the environment for crossbuilding. Change the create_pbuilder function
> that it will write the correct pbuilder config file. Revise the pdebuild_build
> function to make sure that in case of --cross used with the build command
> pbuilder runs with the '--host-arch' option. Throw an error if the pbuilder
> environment was created with the --cross option but the build command gets
> called without the --cross option and vice versa.
>
> Signed-off-by: Christian Teklenborg <chris at linutronix.de>
i am not completely happy with the scanning prioritizing non cross
over cross.
I am not completely sure, if you will get the right result, if you do a normal
build, and then a cross build.
Anyways, lets fix this issue later by deleting the other result dir or
something, if it actually happens.
Please fold this patch with patch 1 and add:
Reviewed-by: Torben Hohn <torben.hohn at linutronix.de>
> ---
> elbepack/db.py | 11 +++-
> elbepack/elbeproject.py | 128 +++++++++++++++++++++++++++----------
> elbepack/projectmanager.py | 17 +++--
> elbepack/soapclient.py | 13 ++--
> 4 files changed, 119 insertions(+), 50 deletions(-)
>
> diff --git a/elbepack/db.py b/elbepack/db.py
> index 8168301c..42ebb83c 100644
> --- a/elbepack/db.py
> +++ b/elbepack/db.py
> @@ -882,10 +882,17 @@ class ElbeDB(object):
> "Repository IsoImage")
>
> # Scan pbuilder/build directory if that exists
> - pbresult_path = os.path.join(p.builddir, "pbuilder", "result")
> + if os.path.exists(os.path.join(p.builddir, "pbuilder", "result")):
> + pbresult_path = os.path.join(p.builddir, "pbuilder", "result")
> + pfile_path = os.path.join("pbuilder", "result")
> + else:
> + pbresult_path = os.path.join(p.builddir, "pbuilder_cross",
> + "result")
> + pfile_path = os.path.join("pbuilder_cross", "result")
> +
> if os.path.isdir(pbresult_path):
> for f in os.listdir(pbresult_path):
> - pfile = os.path.join("pbuilder", "result", f)
> + pfile = os.path.join(pfile_path, f)
> _update_project_file(s, p.builddir, pfile,
> "application/octet-stream",
> "Pbuilder artifact")
> diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py
> index 33627711..4f0dbb56 100644
> --- a/elbepack/elbeproject.py
> +++ b/elbepack/elbeproject.py
> @@ -12,6 +12,7 @@ import os
> import datetime
> import io
> import logging
> +import sys
>
> from elbepack.shellhelper import CommandError, system, do, chroot
>
> @@ -31,6 +32,7 @@ from elbepack.dump import check_full_pkgs
> from elbepack.cdroms import mk_source_cdrom, mk_binary_cdrom
>
> from elbepack.pbuilder import (pbuilder_write_config, pbuilder_write_repo_hook,
> + pbuilder_write_cross_config,
> pbuilder_write_apt_conf)
>
> from elbepack.repomanager import ProjectRepo
> @@ -506,7 +508,7 @@ class ElbeProject (object):
>
> if self.xml.has('target/pbuilder') and not skip_pbuild:
> if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
> - self.create_pbuilder()
> + self.create_pbuilder(cross=False)
> for p in self.xml.node('target/pbuilder'):
> self.pbuild(p)
> # the package might be needed by a following pbuild, so update
> @@ -650,14 +652,31 @@ class ElbeProject (object):
> "pdebuilder"))
>
> # Remove pbuilder/result directory
> - do('rm -rf "%s"' % os.path.join(self.builddir,
> - "pbuilder", "result"))
> + do('rm -rf "%s" "%s"' % (os.path.join(self.builddir,
> + "pbuilder", "result"),
> + os.path.join(self.builddir,
> + "pbuilder_cross","result")))
>
> # Recreate the directories removed
> - do('mkdir -p "%s"' % os.path.join(self.builddir,
> - "pbuilder", "result"))
> -
> - def pdebuild(self, cpuset, profile):
> + if os.path.exists(os.path.join(self.builddir, "pbuilder_cross")):
> + do('mkdir -p "%s"' % os.path.join(self.builddir,
> + "pbuilder_cross","result"))
> + else:
> + do('mkdir -p "%s"' % os.path.join(self.builddir,
> + "pbuilder", "result"))
> +
> + def pdebuild(self, cpuset, profile, cross):
> + cross_pbuilderrc = os.path.join(self.builddir, "cross_pbuilderrc")
> + if cross and not os.path.exists(cross_pbuilderrc):
> + logging.error("Please make sure that you create the pbuilder environment "
> + "with the --cross option if you want to use the build "
> + "command with --cross.")
> + sys.exit(20)
> + if os.path.exists(cross_pbuilderrc) and not cross:
> + logging.error("Please make sure that if you created the pbuilder "
> + "environment without the --cross option, you use the "
> + "build command without --cross too.")
> + sys.exit(20)
> self.pdebuild_init()
>
> pbdir = os.path.join(self.builddir, "pdebuilder", "current")
> @@ -677,10 +696,10 @@ class ElbeProject (object):
> do('tar xfz "%s" -C "%s"' %
> (os.path.join(self.builddir, "current_pdebuild.tar.gz"), pbdir))
>
> - self.pdebuild_build(cpuset, profile)
> + self.pdebuild_build(cpuset, profile, cross)
> self.repo.finalize()
>
> - def pdebuild_build(self, cpuset, profile):
> + def pdebuild_build(self, cpuset, profile, cross):
> # check whether we have to use taskset to run pdebuild
> # this might be useful, when things like java dont
> # work with multithreading
> @@ -692,15 +711,35 @@ class ElbeProject (object):
> cpuset_cmd = ''
>
> try:
> - do('cd "%s"; %s pdebuild --debbuildopts "-j%s -sa" '
> - '--configfile "%s" '
> - '--use-pdebuild-internal --buildresult "%s"' % (
> - os.path.join(self.builddir, "pdebuilder", "current"),
> - cpuset_cmd,
> - cfg['pbuilder_jobs'],
> - os.path.join(self.builddir, "pbuilderrc"),
> - os.path.join(self.builddir, "pbuilder", "result")),
> - env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
> + if cross:
> + do('cd "%s"; dpkg-source -b .; %s '
> + 'pbuilder build --host-arch %s --configfile "%s" '
> + '--basetgz "%s" --buildresult "%s" '
> + '../*.dsc' % (
> + os.path.join(self.builddir, "pdebuilder", "current"),
> + cpuset_cmd, self.arch,
> + os.path.join(self.builddir, "cross_pbuilderrc"),
> + os.path.join(self.builddir, "pbuilder_cross", "base.tgz"),
> + os.path.join(self.builddir, "pbuilder_cross", "result")),
> + env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
> + self.repo.include(os.path.join(self.builddir,
> + "pbuilder_cross",
> + "result",
> + "*.changes"))
> + else:
> + do('cd "%s"; %s pdebuild --debbuildopts "-j%s -sa" '
> + '--configfile "%s" '
> + '--use-pdebuild-internal --buildresult "%s"' % (
> + os.path.join(self.builddir, "pdebuilder", "current"),
> + cpuset_cmd,
> + cfg['pbuilder_jobs'],
> + os.path.join(self.builddir, "pbuilderrc"),
> + os.path.join(self.builddir, "pbuilder", "result")),
> + env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
> + self.repo.include(os.path.join(self.builddir,
> + "pbuilder",
> + "result",
> + "*.changes"))
>
> self.repo.remove(os.path.join(self.builddir,
> "pdebuilder",
> @@ -708,11 +747,9 @@ class ElbeProject (object):
> "debian",
> "control"))
>
> - self.repo.include(os.path.join(self.builddir,
> - "pbuilder", "result", "*.changes"))
> except CommandError:
> logging.exception("Package fails to build.\n"
> - "Please make sure, that the submited package "
> + "Please make sure, that the submitted package "
> "builds in pbuilder")
>
> def update_pbuilder(self):
> @@ -720,30 +757,51 @@ class ElbeProject (object):
> (os.path.join(self.builddir, "pbuilderrc"),
> os.path.join(self.builddir, "aptconfdir")))
>
> - def create_pbuilder(self):
> + def create_pbuilder(self, cross):
> # Remove old pbuilder directory, if it exists
> - do('rm -rf "%s"' % os.path.join(self.builddir, "pbuilder"))
> + do('rm -rf "%s" "%s"' % (os.path.join(self.builddir, "pbuilder"),
> + os.path.join(self.builddir, "pbuilder_cross")))
>
> # make hooks.d and pbuilder directory
> - do('mkdir -p "%s"' %
> - os.path.join(self.builddir, "pbuilder", "hooks.d"))
> - do('mkdir -p "%s"' %
> - os.path.join(self.builddir, "pbuilder", "aptcache"))
> + if cross:
> + do('mkdir -p "%s"' %
> + os.path.join(self.builddir, "pbuilder_cross", "hooks.d"))
> + do('mkdir -p "%s"' %
> + os.path.join(self.builddir, "pbuilder_cross", "aptcache"))
> + else:
> + do('mkdir -p "%s"' %
> + os.path.join(self.builddir, "pbuilder", "hooks.d"))
> + do('mkdir -p "%s"' %
> + os.path.join(self.builddir, "pbuilder", "aptcache"))
> do('mkdir -p "%s"' %
> os.path.join(self.builddir, "aptconfdir", "apt.conf.d"))
>
> # write config files
> - pbuilder_write_config(self.builddir, self.xml)
> + if cross:
> + pbuilder_write_cross_config(self.builddir, self.xml)
> + pbuilder_write_repo_hook(self.builddir, self.xml, cross)
> + do('chmod -R 755 "%s"' %
> + os.path.join(self.builddir, "pbuilder_cross", "hooks.d"))
> + else:
> + pbuilder_write_config(self.builddir, self.xml)
> + pbuilder_write_repo_hook(self.builddir, self.xml, cross)
> + do('chmod -R 755 "%s"' %
> + os.path.join(self.builddir, "pbuilder", "hooks.d"))
> pbuilder_write_apt_conf(self.builddir, self.xml)
> - pbuilder_write_repo_hook(self.builddir, self.xml)
> - do('chmod -R 755 "%s"' %
> - os.path.join(self.builddir, "pbuilder", "hooks.d"))
>
> # Run pbuilder --create
> - do('pbuilder --create --configfile "%s" --aptconfdir "%s" '
> - '--debootstrapopts --include="git gnupg"' %
> - (os.path.join(self.builddir, "pbuilderrc"),
> - os.path.join(self.builddir, "aptconfdir")))
> + if cross:
> + do('pbuilder --create --buildplace "%s" '
> + '--configfile "%s" --aptconfdir "%s" '
> + '--debootstrapopts --include="git gnupg";' %
> + (os.path.join(self.builddir, "pbuilder_cross"),
> + os.path.join(self.builddir, "cross_pbuilderrc"),
> + os.path.join(self.builddir, "aptconfdir")))
> + else:
> + do('pbuilder --create --configfile "%s" --aptconfdir "%s" '
> + '--debootstrapopts --include="git gnupg"' %
> + (os.path.join(self.builddir, "pbuilderrc"),
> + os.path.join(self.builddir, "aptconfdir")))
>
> def sync_xml_to_disk(self):
> try:
> diff --git a/elbepack/projectmanager.py b/elbepack/projectmanager.py
> index f8ed894f..0aca90ff 100644
> --- a/elbepack/projectmanager.py
> +++ b/elbepack/projectmanager.py
> @@ -308,24 +308,26 @@ class ProjectManager(object):
> ep = self._get_current_project(userid, allow_busy=False)
> self.worker.enqueue(UpdatePbuilderJob(ep))
>
> - def build_pbuilder(self, userid):
> + def build_pbuilder(self, userid, cross):
> with self.lock:
> ep = self._get_current_project(userid, allow_busy=False)
> - self.worker.enqueue(CreatePbuilderJob(ep))
> + self.worker.enqueue(CreatePbuilderJob(ep, cross))
>
> - def build_current_pdebuild(self, userid, cpuset, profile):
> + def build_current_pdebuild(self, userid, cpuset, profile, cross):
> with self.lock:
> ep = self._get_current_project(userid, allow_busy=False)
> - if not path.isdir(path.join(ep.builddir, "pbuilder")):
> + if (not path.isdir(path.join(ep.builddir, "pbuilder")) and
> + not path.isdir(path.join(ep.builddir, "pbuilder_cross"))):
> raise InvalidState('No pbuilder exists: run "elbe pbuilder '
> 'create --project %s" first' % ep.builddir)
>
> - self.worker.enqueue(PdebuildJob(ep, cpuset, profile))
> + self.worker.enqueue(PdebuildJob(ep, cpuset, profile, cross))
>
> def set_orig_fname(self, userid, fname):
> with self.lock:
> ep = self._get_current_project(userid, allow_busy=False)
> - if not path.isdir(path.join(ep.builddir, "pbuilder")):
> + if (not path.isdir(path.join(ep.builddir, "pbuilder")) and
> + not path.isdir(path.join(ep.builddir, "pbuilder_cross"))):
> raise InvalidState('No pbuilder exists: run "elbe pbuilder '
> 'create --project %s" first' % ep.builddir)
>
> @@ -335,7 +337,8 @@ class ProjectManager(object):
> def get_orig_fname(self, userid):
> with self.lock:
> ep = self._get_current_project(userid, allow_busy=False)
> - if not path.isdir(path.join(ep.builddir, "pbuilder")):
> + if (not path.isdir(path.join(ep.builddir, "pbuilder")) and
> + not path.isdir(path.join(ep.builddir, "pbuilder_cross"))):
> raise InvalidState('No pbuilder exists: run "elbe pbuilder '
> 'create --project %s" first' % ep.builddir)
>
> diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
> index 586fa513..de159451 100644
> --- a/elbepack/soapclient.py
> +++ b/elbepack/soapclient.py
> @@ -546,7 +546,7 @@ class GetFilesAction(ClientAction):
> nfiles = 0
>
> for f in files[0]:
> - if opt.pbuilder_only and not f.name.startswith('pbuilder'):
> + if (opt.pbuilder_only and not f.name.startswith('pbuilder_cross')):
> continue
>
> if opt.matches and not fnmatch.fnmatch(f.name, opt.matches):
> @@ -722,7 +722,7 @@ class SetPdebuilderAction(ClientAction):
> def execute(self, client, opt, args):
> size = 1024 * 1024
>
> - if len(args) != 2:
> + if len(args) != 2 and len(args) != 3:
> print("usage: elbe control set_pdebuild "
> "<project_dir> <pdebuild file>", file=sys.stderr)
> sys.exit(20)
> @@ -739,7 +739,8 @@ class SetPdebuilderAction(ClientAction):
> if len(bindata) != size:
> break
>
> - client.service.finish_pdebuild(builddir, opt.cpuset, opt.profile)
> + client.service.finish_pdebuild(builddir, opt.cpuset,
> + opt.profile, opt.cross)
>
>
> ClientAction.register(SetPdebuilderAction)
> @@ -752,15 +753,15 @@ class BuildPbuilderAction(ClientAction):
> def __init__(self, node):
> ClientAction.__init__(self, node)
>
> - def execute(self, client, _opt, args):
> - if len(args) != 1:
> + def execute(self, client, opt, args):
> + if len(args) != 1 and len(args) != 2:
> print(
> "usage: elbe control build_pbuilder <project_dir>",
> file=sys.stderr)
> sys.exit(20)
>
> builddir = args[0]
> - client.service.build_pbuilder(builddir)
> + client.service.build_pbuilder(builddir, opt.cross)
>
>
> ClientAction.register(BuildPbuilderAction)
> --
> 2.20.1
>
>
> _______________________________________________
> elbe-devel mailing list
> elbe-devel at linutronix.de
> https://lists.linutronix.de/mailman/listinfo/elbe-devel
--
Torben Hohn
Linutronix GmbH | Bahnhofstrasse 3 | D-88690 Uhldingen-Mühlhofen
Phone: +49 7556 25 999 18; Fax.: +49 7556 25 999 99
Hinweise zum Datenschutz finden Sie hier (Informations on data privacy
can be found here): https://linutronix.de/kontakt/Datenschutz.php
Linutronix GmbH | Firmensitz (Registered Office): Uhldingen-Mühlhofen |
Registergericht (Registration Court): Amtsgericht Freiburg i.Br., HRB700
806 | Geschäftsführer (Managing Directors): Heinz Egger, Thomas Gleixner
More information about the elbe-devel
mailing list