[elbe-devel] [PATCH v4 1/3] pbuilder: adjust create_pbuilder and pdebuild_build

Christian Teklenborg chris at linutronix.de
Thu May 7 14:34:13 CEST 2020


Create a build environment for crossbuilding packages if the 'elbe pbuilder
--cross create' command is called. Now you can run 'elbe pbuilder --cross build'
to use the environment for crossbuilding. Change the create_pbuilder function
that it will create the correct chroot environment. Add the function
pbuilder_write_cross_config to write the correct config file for pbuilder.
Change the pbuilder_write_repo_hook function to take care of the different
directory names from the pbuilder environment.
Revise the pdebuild_build function to make sure that in case of --cross used
with the build command pbuilder runs with the '--host-arch' option. Throw an
error if the pbuilder environment was created with the --cross option but the
build command gets called without the --cross option and vice versa.

Signed-off-by: Christian Teklenborg <chris at linutronix.de>
Reviewed-by: Torben Hohn <torben.hohn at linutronix.de>
---
 elbepack/db.py             |  11 +++-
 elbepack/elbeproject.py    | 128 +++++++++++++++++++++++++++----------
 elbepack/pbuilder.py       |  30 ++++++++-
 elbepack/projectmanager.py |  17 +++--
 elbepack/soapclient.py     |  13 ++--
 5 files changed, 147 insertions(+), 52 deletions(-)

diff --git a/elbepack/db.py b/elbepack/db.py
index 8168301c..42ebb83c 100644
--- a/elbepack/db.py
+++ b/elbepack/db.py
@@ -882,10 +882,17 @@ class ElbeDB(object):
                                           "Repository IsoImage")
 
             # Scan pbuilder/build directory if that exists
-            pbresult_path = os.path.join(p.builddir, "pbuilder", "result")
+            if os.path.exists(os.path.join(p.builddir, "pbuilder", "result")):
+                pbresult_path = os.path.join(p.builddir, "pbuilder", "result")
+                pfile_path = os.path.join("pbuilder", "result")
+            else:
+                pbresult_path = os.path.join(p.builddir, "pbuilder_cross",
+                                             "result")
+                pfile_path = os.path.join("pbuilder_cross", "result")
+
             if os.path.isdir(pbresult_path):
                 for f in os.listdir(pbresult_path):
-                    pfile = os.path.join("pbuilder", "result", f)
+                    pfile = os.path.join(pfile_path, f)
                     _update_project_file(s, p.builddir, pfile,
                                               "application/octet-stream",
                                               "Pbuilder artifact")
diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py
index 33627711..4f0dbb56 100644
--- a/elbepack/elbeproject.py
+++ b/elbepack/elbeproject.py
@@ -12,6 +12,7 @@ import os
 import datetime
 import io
 import logging
+import sys
 
 from elbepack.shellhelper import CommandError, system, do, chroot
 
@@ -31,6 +32,7 @@ from elbepack.dump import check_full_pkgs
 from elbepack.cdroms import mk_source_cdrom, mk_binary_cdrom
 
 from elbepack.pbuilder import (pbuilder_write_config, pbuilder_write_repo_hook,
+                               pbuilder_write_cross_config,
                                pbuilder_write_apt_conf)
 
 from elbepack.repomanager import ProjectRepo
@@ -506,7 +508,7 @@ class ElbeProject (object):
 
         if self.xml.has('target/pbuilder') and not skip_pbuild:
             if not os.path.exists(os.path.join(self.builddir, "pbuilder")):
-                self.create_pbuilder()
+                self.create_pbuilder(cross=False)
             for p in self.xml.node('target/pbuilder'):
                 self.pbuild(p)
                 # the package might be needed by a following pbuild, so update
@@ -650,14 +652,31 @@ class ElbeProject (object):
                                         "pdebuilder"))
 
         # Remove pbuilder/result directory
-        do('rm -rf "%s"' % os.path.join(self.builddir,
-                                        "pbuilder", "result"))
+        do('rm -rf "%s" "%s"' % (os.path.join(self.builddir,
+                                         "pbuilder", "result"),
+                                 os.path.join(self.builddir,
+                                         "pbuilder_cross","result")))
 
         # Recreate the directories removed
-        do('mkdir -p "%s"' % os.path.join(self.builddir,
-                                          "pbuilder", "result"))
-
-    def pdebuild(self, cpuset, profile):
+        if os.path.exists(os.path.join(self.builddir, "pbuilder_cross")):
+            do('mkdir -p "%s"' % os.path.join(self.builddir,
+                                              "pbuilder_cross","result"))
+        else:
+            do('mkdir -p "%s"' % os.path.join(self.builddir,
+                                              "pbuilder", "result"))
+
+    def pdebuild(self, cpuset, profile, cross):
+        cross_pbuilderrc = os.path.join(self.builddir, "cross_pbuilderrc")
+        if cross and not os.path.exists(cross_pbuilderrc):
+            logging.error("Please make sure that you create the pbuilder environment "
+                          "with the --cross option if you want to use the build "
+                          "command with --cross.")
+            sys.exit(20)
+        if os.path.exists(cross_pbuilderrc) and not cross:
+            logging.error("Please make sure that if you created the pbuilder "
+                          "environment without the --cross option, you use the "
+                          "build command without --cross too.")
+            sys.exit(20)
         self.pdebuild_init()
 
         pbdir = os.path.join(self.builddir, "pdebuilder", "current")
@@ -677,10 +696,10 @@ class ElbeProject (object):
         do('tar xfz "%s" -C "%s"' %
            (os.path.join(self.builddir, "current_pdebuild.tar.gz"), pbdir))
 
-        self.pdebuild_build(cpuset, profile)
+        self.pdebuild_build(cpuset, profile, cross)
         self.repo.finalize()
 
-    def pdebuild_build(self, cpuset, profile):
+    def pdebuild_build(self, cpuset, profile, cross):
         # check whether we have to use taskset to run pdebuild
         # this might be useful, when things like java dont
         # work with multithreading
@@ -692,15 +711,35 @@ class ElbeProject (object):
             cpuset_cmd = ''
 
         try:
-            do('cd "%s"; %s pdebuild --debbuildopts "-j%s -sa" '
-               '--configfile "%s" '
-               '--use-pdebuild-internal --buildresult "%s"' % (
-                   os.path.join(self.builddir, "pdebuilder", "current"),
-                   cpuset_cmd,
-                   cfg['pbuilder_jobs'],
-                   os.path.join(self.builddir, "pbuilderrc"),
-                   os.path.join(self.builddir, "pbuilder", "result")),
-               env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
+            if cross:
+                do('cd "%s"; dpkg-source -b .; %s '
+                   'pbuilder build --host-arch %s --configfile "%s" '
+                   '--basetgz "%s" --buildresult "%s" '
+                   '../*.dsc' % (
+                       os.path.join(self.builddir, "pdebuilder", "current"),
+                       cpuset_cmd, self.arch,
+                       os.path.join(self.builddir, "cross_pbuilderrc"),
+                       os.path.join(self.builddir, "pbuilder_cross", "base.tgz"),
+                       os.path.join(self.builddir, "pbuilder_cross", "result")),
+                   env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
+                self.repo.include(os.path.join(self.builddir,
+                                               "pbuilder_cross",
+                                               "result",
+                                               "*.changes"))
+            else:
+                do('cd "%s"; %s pdebuild --debbuildopts "-j%s -sa" '
+                   '--configfile "%s" '
+                   '--use-pdebuild-internal --buildresult "%s"' % (
+                       os.path.join(self.builddir, "pdebuilder", "current"),
+                       cpuset_cmd,
+                       cfg['pbuilder_jobs'],
+                       os.path.join(self.builddir, "pbuilderrc"),
+                       os.path.join(self.builddir, "pbuilder", "result")),
+                   env_add={'DEB_BUILD_PROFILES': profile.replace(",", " ")})
+                self.repo.include(os.path.join(self.builddir,
+                                               "pbuilder",
+                                               "result",
+                                               "*.changes"))
 
             self.repo.remove(os.path.join(self.builddir,
                                           "pdebuilder",
@@ -708,11 +747,9 @@ class ElbeProject (object):
                                           "debian",
                                           "control"))
 
-            self.repo.include(os.path.join(self.builddir,
-                                           "pbuilder", "result", "*.changes"))
         except CommandError:
             logging.exception("Package fails to build.\n"
-                              "Please make sure, that the submited package "
+                              "Please make sure, that the submitted package "
                               "builds in pbuilder")
 
     def update_pbuilder(self):
@@ -720,30 +757,51 @@ class ElbeProject (object):
            (os.path.join(self.builddir, "pbuilderrc"),
             os.path.join(self.builddir, "aptconfdir")))
 
-    def create_pbuilder(self):
+    def create_pbuilder(self, cross):
         # Remove old pbuilder directory, if it exists
-        do('rm -rf "%s"' % os.path.join(self.builddir, "pbuilder"))
+        do('rm -rf "%s" "%s"' % (os.path.join(self.builddir, "pbuilder"),
+                                 os.path.join(self.builddir, "pbuilder_cross")))
 
         # make hooks.d and pbuilder directory
-        do('mkdir -p "%s"' %
-           os.path.join(self.builddir, "pbuilder", "hooks.d"))
-        do('mkdir -p "%s"' %
-           os.path.join(self.builddir, "pbuilder", "aptcache"))
+        if cross:
+            do('mkdir -p "%s"' %
+               os.path.join(self.builddir, "pbuilder_cross", "hooks.d"))
+            do('mkdir -p "%s"' %
+               os.path.join(self.builddir, "pbuilder_cross", "aptcache"))
+        else:
+            do('mkdir -p "%s"' %
+               os.path.join(self.builddir, "pbuilder", "hooks.d"))
+            do('mkdir -p "%s"' %
+               os.path.join(self.builddir, "pbuilder", "aptcache"))
         do('mkdir -p "%s"' %
            os.path.join(self.builddir, "aptconfdir", "apt.conf.d"))
 
         # write config files
-        pbuilder_write_config(self.builddir, self.xml)
+        if cross:
+            pbuilder_write_cross_config(self.builddir, self.xml)
+            pbuilder_write_repo_hook(self.builddir, self.xml, cross)
+            do('chmod -R 755 "%s"' %
+               os.path.join(self.builddir, "pbuilder_cross", "hooks.d"))
+        else:
+            pbuilder_write_config(self.builddir, self.xml)
+            pbuilder_write_repo_hook(self.builddir, self.xml, cross)
+            do('chmod -R 755 "%s"' %
+               os.path.join(self.builddir, "pbuilder", "hooks.d"))
         pbuilder_write_apt_conf(self.builddir, self.xml)
-        pbuilder_write_repo_hook(self.builddir, self.xml)
-        do('chmod -R 755 "%s"' %
-           os.path.join(self.builddir, "pbuilder", "hooks.d"))
 
         # Run pbuilder --create
-        do('pbuilder --create --configfile "%s" --aptconfdir "%s" '
-           '--debootstrapopts --include="git gnupg"' %
-           (os.path.join(self.builddir, "pbuilderrc"),
-            os.path.join(self.builddir, "aptconfdir")))
+        if cross:
+            do('pbuilder --create --buildplace "%s" '
+               '--configfile "%s" --aptconfdir "%s" '
+               '--debootstrapopts --include="git gnupg";' %
+               (os.path.join(self.builddir, "pbuilder_cross"),
+                os.path.join(self.builddir, "cross_pbuilderrc"),
+                os.path.join(self.builddir, "aptconfdir")))
+        else:
+            do('pbuilder --create --configfile "%s" --aptconfdir "%s" '
+               '--debootstrapopts --include="git gnupg"' %
+               (os.path.join(self.builddir, "pbuilderrc"),
+                os.path.join(self.builddir, "aptconfdir")))
 
     def sync_xml_to_disk(self):
         try:
diff --git a/elbepack/pbuilder.py b/elbepack/pbuilder.py
index 6c40a240..e6f0b3cf 100644
--- a/elbepack/pbuilder.py
+++ b/elbepack/pbuilder.py
@@ -62,6 +62,29 @@ def pbuilder_write_config(builddir, xml):
 
     fp.close()
 
+def pbuilder_write_cross_config(builddir, xml):
+    distname = xml.prj.text('suite')
+    pbuilderrc_fname = os.path.join(builddir, "cross_pbuilderrc")
+    fp = open(pbuilderrc_fname, "w")
+
+    fp.write('#!/bin/sh\n')
+    fp.write('set -e\n')
+    fp.write('MIRRORSITE="%s"\n' % xml.get_primary_mirror(False))
+    fp.write('OTHERMIRROR="deb http://127.0.0.1:8080%s/repo %s main"\n' %
+             (builddir, distname))
+    fp.write('BASETGZ="%s"\n' % os.path.join(builddir, 'pbuilder_cross', 'base.tgz'))
+
+    fp.write('DISTRIBUTION="%s"\n' % distname)
+
+    fp.write('BUILDRESULT="%s"\n' %
+             os.path.join(builddir, 'pbuilder_cross', 'result'))
+    fp.write('APTCACHE="%s"\n' %
+             os.path.join(builddir, 'pbuilder_cross','aptcache'))
+    fp.write('HOOKDIR="%s"\n' % os.path.join(builddir, 'pbuilder_cross', 'hooks.d'))
+    fp.write('PBUILDERSATISFYDEPENDSCMD='
+             '/usr/lib/pbuilder/pbuilder-satisfydepends-apt\n')
+    fp.close()
+
 
 def pbuilder_write_apt_conf(builddir, xml):
 
@@ -107,9 +130,12 @@ def mirror_script_add_key_url(mirror, key_url):
     return mirror_script_add_key_text(mirror, key_text)
 
 
-def pbuilder_write_repo_hook(builddir, xml):
+def pbuilder_write_repo_hook(builddir, xml, cross):
 
-    pbuilder_hook_dir = os.path.join(builddir, "pbuilder", "hooks.d")
+    if cross:
+        pbuilder_hook_dir = os.path.join(builddir, "pbuilder_cross", "hooks.d")
+    else:
+        pbuilder_hook_dir = os.path.join(builddir, "pbuilder", "hooks.d")
 
     with open(os.path.join(pbuilder_hook_dir, "H10elbe_apt_update"), "w") as f:
         f.write("#!/bin/sh\napt update\n")
diff --git a/elbepack/projectmanager.py b/elbepack/projectmanager.py
index f8ed894f..0aca90ff 100644
--- a/elbepack/projectmanager.py
+++ b/elbepack/projectmanager.py
@@ -308,24 +308,26 @@ class ProjectManager(object):
             ep = self._get_current_project(userid, allow_busy=False)
             self.worker.enqueue(UpdatePbuilderJob(ep))
 
-    def build_pbuilder(self, userid):
+    def build_pbuilder(self, userid, cross):
         with self.lock:
             ep = self._get_current_project(userid, allow_busy=False)
-            self.worker.enqueue(CreatePbuilderJob(ep))
+            self.worker.enqueue(CreatePbuilderJob(ep, cross))
 
-    def build_current_pdebuild(self, userid, cpuset, profile):
+    def build_current_pdebuild(self, userid, cpuset, profile, cross):
         with self.lock:
             ep = self._get_current_project(userid, allow_busy=False)
-            if not path.isdir(path.join(ep.builddir, "pbuilder")):
+            if (not path.isdir(path.join(ep.builddir, "pbuilder")) and
+                    not path.isdir(path.join(ep.builddir, "pbuilder_cross"))):
                 raise InvalidState('No pbuilder exists: run "elbe pbuilder '
                                    'create --project %s" first' % ep.builddir)
 
-            self.worker.enqueue(PdebuildJob(ep, cpuset, profile))
+            self.worker.enqueue(PdebuildJob(ep, cpuset, profile, cross))
 
     def set_orig_fname(self, userid, fname):
         with self.lock:
             ep = self._get_current_project(userid, allow_busy=False)
-            if not path.isdir(path.join(ep.builddir, "pbuilder")):
+            if (not path.isdir(path.join(ep.builddir, "pbuilder")) and
+                    not path.isdir(path.join(ep.builddir, "pbuilder_cross"))):
                 raise InvalidState('No pbuilder exists: run "elbe pbuilder '
                                    'create --project %s" first' % ep.builddir)
 
@@ -335,7 +337,8 @@ class ProjectManager(object):
     def get_orig_fname(self, userid):
         with self.lock:
             ep = self._get_current_project(userid, allow_busy=False)
-            if not path.isdir(path.join(ep.builddir, "pbuilder")):
+            if (not path.isdir(path.join(ep.builddir, "pbuilder")) and
+                    not path.isdir(path.join(ep.builddir, "pbuilder_cross"))):
                 raise InvalidState('No pbuilder exists: run "elbe pbuilder '
                                    'create --project %s" first' % ep.builddir)
 
diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
index 586fa513..de159451 100644
--- a/elbepack/soapclient.py
+++ b/elbepack/soapclient.py
@@ -546,7 +546,7 @@ class GetFilesAction(ClientAction):
         nfiles = 0
 
         for f in files[0]:
-            if opt.pbuilder_only and not f.name.startswith('pbuilder'):
+            if (opt.pbuilder_only and not f.name.startswith('pbuilder_cross')):
                 continue
 
             if opt.matches and not fnmatch.fnmatch(f.name, opt.matches):
@@ -722,7 +722,7 @@ class SetPdebuilderAction(ClientAction):
     def execute(self, client, opt, args):
         size = 1024 * 1024
 
-        if len(args) != 2:
+        if len(args) != 2 and len(args) != 3:
             print("usage: elbe control set_pdebuild "
                   "<project_dir> <pdebuild file>", file=sys.stderr)
             sys.exit(20)
@@ -739,7 +739,8 @@ class SetPdebuilderAction(ClientAction):
             if len(bindata) != size:
                 break
 
-        client.service.finish_pdebuild(builddir, opt.cpuset, opt.profile)
+        client.service.finish_pdebuild(builddir, opt.cpuset,
+                                       opt.profile, opt.cross)
 
 
 ClientAction.register(SetPdebuilderAction)
@@ -752,15 +753,15 @@ class BuildPbuilderAction(ClientAction):
     def __init__(self, node):
         ClientAction.__init__(self, node)
 
-    def execute(self, client, _opt, args):
-        if len(args) != 1:
+    def execute(self, client, opt, args):
+        if len(args) != 1 and len(args) != 2:
             print(
                 "usage: elbe control build_pbuilder <project_dir>",
                 file=sys.stderr)
             sys.exit(20)
 
         builddir = args[0]
-        client.service.build_pbuilder(builddir)
+        client.service.build_pbuilder(builddir, opt.cross)
 
 
 ClientAction.register(BuildPbuilderAction)
-- 
2.20.1




More information about the elbe-devel mailing list