[elbe-devel] [PATCH] build_sysroot: build sysroot in seperate buildenv
John Ogness
john.ogness at linutronix.de
Fri Jul 20 16:07:07 CEST 2018
On 2018-07-12, Manuel Traut <manut at linutronix.de> wrote:
> instead of using the same buildenv for creating the target RFS and
> building the sysroot use a seperate buildenv for the sysroot. So the
> target RFS cant contain -dev packages if built after the sysroot.
>
> Since there are two 'Buildenv' instances now. sysroot and chroot
> each should have its own rpcaptcache. So store the rpcaptcache reference
> inside Buildenv instead of ElbeProject.
>
> Signed-off-by: Manuel Traut <manut at linutronix.de>
If the debian/control changes are not included:
Reviewed-by: John Ogness <john.ogness at linutronix.de>
> ---
> debian/control | 9 ++++
> elbepack/elbeproject.py | 95 +++++++++++++++++++++++++----------------
> elbepack/rfs.py | 1 +
> 3 files changed, 68 insertions(+), 37 deletions(-)
>
> diff --git a/debian/control b/debian/control
> index 158df1ba..01ead0e6 100644
> --- a/debian/control
> +++ b/debian/control
??? Were the debian/control changes accidentally included in the patch?
> @@ -24,6 +24,15 @@ Build-Depends: debhelper (>= 9),
> Standards-Version: 3.9.6
> Homepage: http://elbe-rfs.org
>
> +Package: elbe-cd2aptly
> +Section: text
> +Architecture: all
> +Depends: ${misc:Depends}, fuseiso9660, aptly
> +Description: insert packages from elbe generated cdroms into aptly repos
> + iso images generated by elbe builds can be added to a aptly repository to
> + use local mirrors without internet connection for rebuilding the same image
> + again
> +
> Package: elbe-doc
> Section: doc
> Architecture: all
> diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py
> index be23410d..6a8090ff 100644
> --- a/elbepack/elbeproject.py
> +++ b/elbepack/elbeproject.py
> @@ -119,6 +119,7 @@ class ElbeProject (object):
> self.builddir = os.path.abspath(str(builddir))
> self.chrootpath = os.path.join(self.builddir, "chroot")
> self.targetpath = os.path.join(self.builddir, "target")
> + self.sysrootpath = os.path.join(self.builddir, "sysroot")
> self.sdkpath = os.path.join(self.builddir, "sdk")
> self.validationpath = os.path.join(self.builddir, "validation.txt")
>
> @@ -191,6 +192,12 @@ class ElbeProject (object):
> else:
> self.targetfs = None
>
> + # dont create sysroot instance, it should be build from scratch
> + # each time, because the pkglist including the -dev packages is
> + # tracked nowhere.
> + self.sysrootenv = None
> +
> +
> def build_chroottarball(self):
> self.log.do("tar cJf %s/chroot.tar.xz \
> --exclude=./tmp/* --exclude=./dev/* \
> @@ -223,8 +230,18 @@ class ElbeProject (object):
>
> def build_sysroot(self):
>
> - if not self.buildenv:
> - self.build()
> + self.log.do('rm -rf %s; mkdir "%s"' % (self.sysrootpath,
> + self.sysrootpath))
> +
> + self.sysrootenv = BuildEnv(self.xml,
> + self.log,
> + self.sysrootpath,
> + clean=True)
> + # Import keyring
> + self.sysrootenv.import_keys()
> + self.log.printo("Keys imported")
> +
> + self.install_packages(self.sysrootenv, buildenv=False)
>
> # ignore packages from debootstrap
> ignore_pkgs = [p.et.text for p in self.xml.node("debootstrappkgs")]
> @@ -233,45 +250,44 @@ class ElbeProject (object):
> ignore_dev_pkgs = [p.et.text for p in self.xml.node(
> "target/pkg-blacklist/sysroot")]
>
> - with self.buildenv:
> + with self.sysrootenv:
> try:
> - self.get_rpcaptcache().update()
> + self.get_rpcaptcache(env=self.sysrootenv).update()
> except Exception as e:
> raise AptCacheUpdateError(e)
>
> try:
> - self.get_rpcaptcache().mark_install_devpkgs(set(ignore_pkgs),
> + self.get_rpcaptcache(env=self.sysrootenv).mark_install_devpkgs(set(ignore_pkgs),
> set(ignore_dev_pkgs))
> except SystemError as e:
> self.log.printo("mark install devpkgs failed: %s" % str(e))
> try:
> - self.get_rpcaptcache().commit()
> + self.get_rpcaptcache(env=self.sysrootenv).commit()
> except SystemError as e:
> self.log.printo("commiting changes failed: %s" % str(e))
> raise AptCacheCommitError(str(e))
>
> + try:
> + self.sysrootenv.rfs.dump_elbeversion(self.xml)
> + except IOError:
> + self.log.printo("dump elbeversion into sysroot failed")
> +
> sysrootfilelist = os.path.join(self.builddir, "sysroot-filelist")
>
> - with self.buildenv.rfs:
> + with self.sysrootenv.rfs:
> self.log.do("chroot %s /usr/bin/symlinks -cr /usr/lib" %
> - self.chrootpath)
> + self.sysrootpath)
>
> paths = self.get_sysroot_paths()
>
> self.log.do("rm %s" % sysrootfilelist, allow_fail=True)
> - os.chdir(self.chrootpath)
> + os.chdir(self.sysrootpath)
> for p in paths:
> self.log.do('find -path "%s" >> %s' % (p, sysrootfilelist))
>
> self.log.do("tar cfJ %s/sysroot.tar.xz -C %s -T %s" %
> - (self.builddir, self.chrootpath, sysrootfilelist))
> + (self.builddir, self.sysrootpath, sysrootfilelist))
>
> - # chroot is invalid after adding all the -dev packages
> - # it shouldn't be used to create an incremental image
> - #
> - # We only remove /etc/elbe_version here.
> - # So we can still elbe chroot into the fs
> - self.buildenv.rfs.remove("/etc/elbe_version", noerr=True)
>
> def build_sdk(self):
> triplet = self.xml.defs["triplet"]
> @@ -377,7 +393,7 @@ class ElbeProject (object):
>
> # Install packages
> if not skip_pkglist:
> - self.install_packages()
> + self.install_packages(self.buildenv)
>
> try:
> self.buildenv.rfs.dump_elbeversion(self.xml)
> @@ -421,7 +437,7 @@ class ElbeProject (object):
>
> # install packages for buildenv
> if not skip_pkglist:
> - self.install_packages(buildenv=True)
> + self.install_packages(self.buildenv, buildenv=True)
>
> # Write source.xml
> try:
> @@ -652,19 +668,23 @@ class ElbeProject (object):
> except MemoryError:
> self.log.printo("write source.xml failed (archive to huge?)")
>
> - def get_rpcaptcache(self):
> - if self._rpcaptcache is None:
> - self._rpcaptcache = get_rpcaptcache(
> - self.buildenv.rfs,
> + def get_rpcaptcache(self, env=None):
> + if not env:
> + env = self.buildenv
> + if env._rpcaptcache is None:
> + env._rpcaptcache = get_rpcaptcache(
> + env.rfs,
> self.log.fp.name,
> self.arch,
> self.rpcaptcache_notifier,
> self.xml.prj.has('norecommend'),
> self.xml.prj.has('noauth'))
> - return self._rpcaptcache
> + return env._rpcaptcache
>
> - def drop_rpcaptcache(self):
> - self._rpcaptcache = None
> + def drop_rpcaptcache(self, env=None):
> + if not env:
> + env = self.buildenv
> + env._rpcaptcache = None
>
> def has_full_buildenv(self):
> if os.path.exists(self.chrootpath):
> @@ -724,20 +744,21 @@ class ElbeProject (object):
> self.log.printo("report timestamp: " +
> datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
>
> - def install_packages(self, buildenv=False):
> - with self.buildenv:
> + def install_packages(self, target, buildenv=False):
> + with target:
> # First update the apt cache
> try:
> - self.get_rpcaptcache().update()
> + self.get_rpcaptcache(env=target).update()
> except Exception as e:
> raise AptCacheUpdateError(e)
>
> # Then dump the debootstrap packages
> - if self.buildenv.fresh_debootstrap:
> - if self.buildenv.need_dumpdebootstrap:
> - dump_debootstrappkgs(self.xml, self.get_rpcaptcache())
> + if target.fresh_debootstrap:
> + if target.need_dumpdebootstrap:
> + dump_debootstrappkgs(self.xml,
> + self.get_rpcaptcache(env=target))
> dump_initvmpkgs(self.xml)
> - self.buildenv.need_dumpdebootstrap = False
> + target.need_dumpdebootstrap = False
> source = self.xml
> try:
> initxml = ElbeXML(
> @@ -772,7 +793,7 @@ class ElbeProject (object):
>
> # Seed /etc, we need /etc/hosts for hostname -f to work correctly
> if not buildenv:
> - self.buildenv.seed_etc()
> + target.seed_etc()
>
> # remove all non-essential packages to ensure that on a incremental
> # build packages can be removed
> @@ -780,15 +801,15 @@ class ElbeProject (object):
> for p in self.xml.node("debootstrappkgs"):
> debootstrap_pkgs.append(p.et.text)
>
> - pkgs = self.buildenv.xml.get_target_packages() + debootstrap_pkgs
> + pkgs = target.xml.get_target_packages() + debootstrap_pkgs
>
> if buildenv:
> - pkgs = pkgs + self.buildenv.xml.get_buildenv_packages()
> + pkgs = pkgs + target.xml.get_buildenv_packages()
>
> # Now install requested packages
> for p in pkgs:
> try:
> - self.get_rpcaptcache().mark_install(p, None)
> + self.get_rpcaptcache(env=target).mark_install(p, None)
> except KeyError:
> self.log.printo("No Package " + p)
> except SystemError:
> @@ -800,7 +821,7 @@ class ElbeProject (object):
> # self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs)
>
> try:
> - self.get_rpcaptcache().commit()
> + self.get_rpcaptcache(env=target).commit()
> except SystemError as e:
> self.log.printo("commiting changes failed: %s" % str(e))
> raise AptCacheCommitError(str(e))
> diff --git a/elbepack/rfs.py b/elbepack/rfs.py
> index cfee1a17..e5527b1c 100644
> --- a/elbepack/rfs.py
> +++ b/elbepack/rfs.py
> @@ -28,6 +28,7 @@ class BuildEnv ():
> self.xml = xml
> self.log = log
> self.path = path
> + self._rpcaptcache = None
>
> self.rfs = BuildImgFs(path, xml.defs["userinterpr"])
More information about the elbe-devel
mailing list