[elbe-devel] [PATCH v3 2/4] pbuilder: adjust pdebuild_build and create_pbuilder

Torben Hohn torben.hohn at linutronix.de
Wed Apr 29 16:06:23 CEST 2020


On Mon, Apr 27, 2020 at 04:35:42PM +0200, Christian Teklenborg wrote:
> Create a build environment for crossbuilding packages if the 'elbe pbuilder
> --cross create' command is called. Now you can run 'elbe pbuilder --cross build'
> to use the environment for crossbuilding. Change the create_pbuilder function
> that it will write the correct pbuilder config file. Revise the pdebuild_build
> function to make sure that in case of --cross used with the build command
> pbuilder runs with the '--host-arch' option. Throw an error if the pbuilder
> environment was created with the --cross option but the build command gets
> called without the --cross option and vice versa.
> 
> Signed-off-by: Christian Teklenborg <chris at linutronix.de>

i am not completely happy with the scanning prioritizing non cross
over cross.

I am not completely sure, if you will get the right result, if you do a normal
build, and then a cross build. 

Anyways, lets fix this issue later by deleting the other result dir or
something, if it actually happens.

Please fold this patch with patch 1 and add:

Reviewed-by: Torben Hohn <torben.hohn at linutronix.de>

> ---
>  elbepack/db.py             |  11 +++-
>  elbepack/elbeproject.py    | 128 +++++++++++++++++++++++++++----------
>  elbepack/projectmanager.py |  17 +++--
>  elbepack/soapclient.py     |  13 ++--
>  4 files changed, 119 insertions(+), 50 deletions(-)
> 
> diff --git a/elbepack/db.py b/elbepack/db.py
> index 8168301c..42ebb83c 100644
> --- a/elbepack/db.py
> +++ b/elbepack/db.py
> @@ -882,10 +882,17 @@ class ElbeDB(object):
>                                            "Repository IsoImage")
>  
>              # Scan pbuilder/build directory if that exists
> -            pbresult_path = os.path.join(p.builddir, "pbuilder", "result")
> +            if os.path.exists(os.path.join(p.builddir, "pbuilder", "result")):
> +                pbresult_path = os.path.join(p.builddir, "pbuilder", "result")
> +                pfile_path = os.path.join("pbuilder", "result")
> +            else:
> +                pbresult_path = os.path.join(p.builddir, "pbuilder_cross",
> +                                             "result")
> +                pfile_path = os.path.join("pbuilder_cross", "result")
> +
>              if os.path.isdir(pbresult_path):
>                  for f in os.listdir(pbresult_path):
> -                    pfile = os.path.join("pbuilder", "result", f)
> +                    pfile = os.path.join(pfile_path, f)
>                      _update_project_file(s, p.builddir, pfile,
>                                                "application/octet-stream",
>                                                "Pbuilder artifact")
> diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py
> index 33627711..4f0dbb56 100644
> --- a/elbepack/elbeproject.py
> +++ b/elbepack/elbeproject.py
> @@ -12,6 +12,7 @@ import os
>  import datetime
>  import io
>  import logging
> +import sys
>  
>  from elbepack.shellhelper import CommandError, system, do, chroot
>  
> @@ -31,6 +32,7 @@ from elbepack.dump import check_full_pkgs
>  from elbepack.cdroms import mk_source_cdrom, mk_binary_cdrom
>  
>  from elbepack.pbuilder import (pbuilder_write_config, pbuilder_write_repo_hook,
> +                               pbuilder_write_cross_config,
>                                 pbuilder_write_apt_conf)
>  
>  from elbepack.repomanager import ProjectRepo
> @@ -506,7 +508,7 @@ class ElbeProject (object):
>  
>          if self.xml.has('target/pbuilder') and not skip_pbuild:
>              if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
> -                self.create_pbuilder()
> +                self.create_pbuilder(cross=False)
>              for p in self.xml.node('target/pbuilder'):
>                  self.pbuild(p)
>                  # the package might be needed by a following pbuild, so update
> @@ -650,14 +652,31 @@ class ElbeProject (object):
>                                          "pdebuilder"))
>  
>          # Remove pbuilder/result directory
> -        do('rm -rf "%s"' % os.path.join(self.builddir,
> -                                        "pbuilder", "result"))
> +        do('rm -rf "%s" "%s"' % (os.path.join(self.builddir,
> +                                         "pbuilder", "result"),
> +                                 os.path.join(self.builddir,
> +                                         "pbuilder_cross","result")))
>  
>          # Recreate the directories removed
> -        do('mkdir -p "%s"' % os.path.join(self.builddir,
> -                                          "pbuilder", "result"))
> -
> -    def pdebuild(self, cpuset, profile):
> +        if os.path.exists(os.path.join(self.builddir, "pbuilder_cross")):
> +            do('mkdir -p "%s"' % os.path.join(self.builddir,
> +                                              "pbuilder_cross","result"))
> +        else:
> +            do('mkdir -p "%s"' % os.path.join(self.builddir,
> +                                              "pbuilder", "result"))
> +
> +    def pdebuild(self, cpuset, profile, cross):
> +        cross_pbuilderrc = os.path.join(self.builddir, "cross_pbuilderrc")
> +        if cross and not os.path.exists(cross_pbuilderrc):
> +            logging.error("Please make sure that you create the pbuilder environment "
> +                          "with the --cross option if you want to use the build "
> +                          "command with --cross.")
> +            sys.exit(20)
> +        if os.path.exists(cross_pbuilderrc) and not cross:
> +            logging.error("Please make sure that if you created the pbuilder "
> +                          "environment without the --cross option, you use the "
> +                          "build command without --cross too.")
> +            sys.exit(20)
>          self.pdebuild_init()
>  
>          pbdir = os.path.join(self.builddir, "pdebuilder", "current")
> @@ -677,10 +696,10 @@ class ElbeProject (object):
>          do('tar xfz "%s" -C "%s"' %
>             (os.path.join(self.builddir, "current_pdebuild.tar.gz"), pbdir))
>  
> -        self.pdebuild_build(cpuset, profile)
> +        self.pdebuild_build(cpuset, profile, cross)
>          self.repo.finalize()
>  
> -    def pdebuild_build(self, cpuset, profile):
> +    def pdebuild_build(self, cpuset, profile, cross):
>          # check whether we have to use taskset to run pdebuild
>          # this might be useful, when things like java dont
>          # work with multithreading
> @@ -692,15 +711,35 @@ class ElbeProject (object):
>              cpuset_cmd = ''
>  
>          try:
> -            do('cd "%s"; %s pdebuild --debbuildopts "-j%s -sa" '
> -               '--configfile "%s" '
> -               '--use-pdebuild-internal --buildresult "%s"' % (
> -                   os.path.join(self.builddir, "pdebuilder", "current"),
> -                   cpuset_cmd,
> -                   cfg['pbuilder_jobs'],
> -                   os.path.join(self.builddir, "pbuilderrc"),
> -                   os.path.join(self.builddir, "pbuilder", "result")),
> -               env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
> +            if cross:
> +                do('cd "%s"; dpkg-source -b .; %s '
> +                   'pbuilder build --host-arch %s --configfile "%s" '
> +                   '--basetgz "%s" --buildresult "%s" '
> +                   '../*.dsc' % (
> +                       os.path.join(self.builddir, "pdebuilder", "current"),
> +                       cpuset_cmd, self.arch,
> +                       os.path.join(self.builddir, "cross_pbuilderrc"),
> +                       os.path.join(self.builddir, "pbuilder_cross", "base.tgz"),
> +                       os.path.join(self.builddir, "pbuilder_cross", "result")),
> +                   env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
> +                self.repo.include(os.path.join(self.builddir,
> +                                               "pbuilder_cross",
> +                                               "result",
> +                                               "*.changes"))
> +            else:
> +                do('cd "%s"; %s pdebuild --debbuildopts "-j%s -sa" '
> +                   '--configfile "%s" '
> +                   '--use-pdebuild-internal --buildresult "%s"' % (
> +                       os.path.join(self.builddir, "pdebuilder", "current"),
> +                       cpuset_cmd,
> +                       cfg['pbuilder_jobs'],
> +                       os.path.join(self.builddir, "pbuilderrc"),
> +                       os.path.join(self.builddir, "pbuilder", "result")),
> +                   env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
> +                self.repo.include(os.path.join(self.builddir,
> +                                               "pbuilder",
> +                                               "result",
> +                                               "*.changes"))
>  
>              self.repo.remove(os.path.join(self.builddir,
>                                            "pdebuilder",
> @@ -708,11 +747,9 @@ class ElbeProject (object):
>                                            "debian",
>                                            "control"))
>  
> -            self.repo.include(os.path.join(self.builddir,
> -                                           "pbuilder", "result", "*.changes"))
>          except CommandError:
>              logging.exception("Package fails to build.\n"
> -                              "Please make sure, that the submited package "
> +                              "Please make sure, that the submitted package "
>                                "builds in pbuilder")
>  
>      def update_pbuilder(self):
> @@ -720,30 +757,51 @@ class ElbeProject (object):
>             (os.path.join(self.builddir, "pbuilderrc"),
>              os.path.join(self.builddir, "aptconfdir")))
>  
> -    def create_pbuilder(self):
> +    def create_pbuilder(self, cross):
>          # Remove old pbuilder directory, if it exists
> -        do('rm -rf "%s"' % os.path.join(self.builddir, "pbuilder"))
> +        do('rm -rf "%s" "%s"' % (os.path.join(self.builddir, "pbuilder"),
> +                                 os.path.join(self.builddir, "pbuilder_cross")))
>  
>          # make hooks.d and pbuilder directory
> -        do('mkdir -p "%s"' %
> -           os.path.join(self.builddir, "pbuilder", "hooks.d"))
> -        do('mkdir -p "%s"' %
> -           os.path.join(self.builddir, "pbuilder", "aptcache"))
> +        if cross:
> +            do('mkdir -p "%s"' %
> +               os.path.join(self.builddir, "pbuilder_cross", "hooks.d"))
> +            do('mkdir -p "%s"' %
> +               os.path.join(self.builddir, "pbuilder_cross", "aptcache"))
> +        else:
> +            do('mkdir -p "%s"' %
> +               os.path.join(self.builddir, "pbuilder", "hooks.d"))
> +            do('mkdir -p "%s"' %
> +               os.path.join(self.builddir, "pbuilder", "aptcache"))
>          do('mkdir -p "%s"' %
>             os.path.join(self.builddir, "aptconfdir", "apt.conf.d"))
>  
>          # write config files
> -        pbuilder_write_config(self.builddir, self.xml)
> +        if cross:
> +            pbuilder_write_cross_config(self.builddir, self.xml)
> +            pbuilder_write_repo_hook(self.builddir, self.xml, cross)
> +            do('chmod -R 755 "%s"' %
> +               os.path.join(self.builddir, "pbuilder_cross", "hooks.d"))
> +        else:
> +            pbuilder_write_config(self.builddir, self.xml)
> +            pbuilder_write_repo_hook(self.builddir, self.xml, cross)
> +            do('chmod -R 755 "%s"' %
> +               os.path.join(self.builddir, "pbuilder", "hooks.d"))
>          pbuilder_write_apt_conf(self.builddir, self.xml)
> -        pbuilder_write_repo_hook(self.builddir, self.xml)
> -        do('chmod -R 755 "%s"' %
> -           os.path.join(self.builddir, "pbuilder", "hooks.d"))
>  
>          # Run pbuilder --create
> -        do('pbuilder --create --configfile "%s" --aptconfdir "%s" '
> -           '--debootstrapopts --include="git gnupg"' %
> -           (os.path.join(self.builddir, "pbuilderrc"),
> -            os.path.join(self.builddir, "aptconfdir")))
> +        if cross:
> +            do('pbuilder --create --buildplace "%s" '
> +               '--configfile "%s" --aptconfdir "%s" '
> +               '--debootstrapopts --include="git gnupg";' %
> +               (os.path.join(self.builddir, "pbuilder_cross"),
> +                os.path.join(self.builddir, "cross_pbuilderrc"),
> +                os.path.join(self.builddir, "aptconfdir")))
> +        else:
> +            do('pbuilder --create --configfile "%s" --aptconfdir "%s" '
> +               '--debootstrapopts --include="git gnupg"' %
> +               (os.path.join(self.builddir, "pbuilderrc"),
> +                os.path.join(self.builddir, "aptconfdir")))
>  
>      def sync_xml_to_disk(self):
>          try:
> diff --git a/elbepack/projectmanager.py b/elbepack/projectmanager.py
> index f8ed894f..0aca90ff 100644
> --- a/elbepack/projectmanager.py
> +++ b/elbepack/projectmanager.py
> @@ -308,24 +308,26 @@ class ProjectManager(object):
>              ep = self._get_current_project(userid, allow_busy=False)
>              self.worker.enqueue(UpdatePbuilderJob(ep))
>  
> -    def build_pbuilder(self, userid):
> +    def build_pbuilder(self, userid, cross):
>          with self.lock:
>              ep = self._get_current_project(userid, allow_busy=False)
> -            self.worker.enqueue(CreatePbuilderJob(ep))
> +            self.worker.enqueue(CreatePbuilderJob(ep, cross))
>  
> -    def build_current_pdebuild(self, userid, cpuset, profile):
> +    def build_current_pdebuild(self, userid, cpuset, profile, cross):
>          with self.lock:
>              ep = self._get_current_project(userid, allow_busy=False)
> -            if not path.isdir(path.join(ep.builddir, "pbuilder")):
> +            if (not path.isdir(path.join(ep.builddir, "pbuilder")) and
> +                    not path.isdir(path.join(ep.builddir, "pbuilder_cross"))):
>                  raise InvalidState('No pbuilder exists: run "elbe pbuilder '
>                                     'create --project %s" first' % ep.builddir)
>  
> -            self.worker.enqueue(PdebuildJob(ep, cpuset, profile))
> +            self.worker.enqueue(PdebuildJob(ep, cpuset, profile, cross))
>  
>      def set_orig_fname(self, userid, fname):
>          with self.lock:
>              ep = self._get_current_project(userid, allow_busy=False)
> -            if not path.isdir(path.join(ep.builddir, "pbuilder")):
> +            if (not path.isdir(path.join(ep.builddir, "pbuilder")) and
> +                    not path.isdir(path.join(ep.builddir, "pbuilder_cross"))):
>                  raise InvalidState('No pbuilder exists: run "elbe pbuilder '
>                                     'create --project %s" first' % ep.builddir)
>  
> @@ -335,7 +337,8 @@ class ProjectManager(object):
>      def get_orig_fname(self, userid):
>          with self.lock:
>              ep = self._get_current_project(userid, allow_busy=False)
> -            if not path.isdir(path.join(ep.builddir, "pbuilder")):
> +            if (not path.isdir(path.join(ep.builddir, "pbuilder")) and
> +                    not path.isdir(path.join(ep.builddir, "pbuilder_cross"))):
>                  raise InvalidState('No pbuilder exists: run "elbe pbuilder '
>                                     'create --project %s" first' % ep.builddir)
>  
> diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
> index 586fa513..de159451 100644
> --- a/elbepack/soapclient.py
> +++ b/elbepack/soapclient.py
> @@ -546,7 +546,7 @@ class GetFilesAction(ClientAction):
>          nfiles = 0
>  
>          for f in files[0]:
> -            if opt.pbuilder_only and not f.name.startswith('pbuilder'):
> +            if (opt.pbuilder_only and not f.name.startswith('pbuilder_cross')):
>                  continue
>  
>              if opt.matches and not fnmatch.fnmatch(f.name, opt.matches):
> @@ -722,7 +722,7 @@ class SetPdebuilderAction(ClientAction):
>      def execute(self, client, opt, args):
>          size = 1024 * 1024
>  
> -        if len(args) != 2:
> +        if len(args) != 2 and len(args) != 3:
>              print("usage: elbe control set_pdebuild "
>                    "<project_dir> <pdebuild file>", file=sys.stderr)
>              sys.exit(20)
> @@ -739,7 +739,8 @@ class SetPdebuilderAction(ClientAction):
>              if len(bindata) != size:
>                  break
>  
> -        client.service.finish_pdebuild(builddir, opt.cpuset, opt.profile)
> +        client.service.finish_pdebuild(builddir, opt.cpuset,
> +                                       opt.profile, opt.cross)
>  
>  
>  ClientAction.register(SetPdebuilderAction)
> @@ -752,15 +753,15 @@ class BuildPbuilderAction(ClientAction):
>      def __init__(self, node):
>          ClientAction.__init__(self, node)
>  
> -    def execute(self, client, _opt, args):
> -        if len(args) != 1:
> +    def execute(self, client, opt, args):
> +        if len(args) != 1 and len(args) != 2:
>              print(
>                  "usage: elbe control build_pbuilder <project_dir>",
>                  file=sys.stderr)
>              sys.exit(20)
>  
>          builddir = args[0]
> -        client.service.build_pbuilder(builddir)
> +        client.service.build_pbuilder(builddir, opt.cross)
>  
>  
>  ClientAction.register(BuildPbuilderAction)
> -- 
> 2.20.1
> 
> 
> _______________________________________________
> elbe-devel mailing list
> elbe-devel at linutronix.de
> https://lists.linutronix.de/mailman/listinfo/elbe-devel

-- 
Torben Hohn
Linutronix GmbH | Bahnhofstrasse 3 | D-88690 Uhldingen-Mühlhofen
Phone: +49 7556 25 999 18; Fax.: +49 7556 25 999 99

Hinweise zum Datenschutz finden Sie hier (Informations on data privacy 
can be found here): https://linutronix.de/kontakt/Datenschutz.php

Linutronix GmbH | Firmensitz (Registered Office): Uhldingen-Mühlhofen | 
Registergericht (Registration Court): Amtsgericht Freiburg i.Br., HRB700 
806 | Geschäftsführer (Managing Directors): Heinz Egger, Thomas Gleixner



More information about the elbe-devel mailing list