[elbe-devel] [PATCH v2 05/22] virtapt: massive refactoring to make it work again
Bastian Germann
bage at linutronix.de
Tue Jul 2 16:50:16 CEST 2019
> virtapt went through a big refactoring, for the get_uri function for
> the sdk. The sdk is implemented differently now, and the single remaining
> user of virtapt is "elbe check_updates".
>
> The result was, that check update did not work anymore.
>
> Clean it up and make <raw-key> work.
>
> - use TmpdirFilesystem (removes os.path.join() usage)
> - make constructor take an ElbeXML object
> - drop the old gpg code, and add code supporting <raw-key>
> - do not rely on <sources_list> and <apt_prefs>, but generate
> them using the same code that a normal elbe build uses
>
> adjust check_update (the sole user of virtapt) to the new constructor
> signature.
>
> Signed-off-by: Torben Hohn <torben.hohn at linutronix.de>
Reviewed-by: Bastian Germann <bage at linutronix.de>
> ---
> elbepack/commands/check_updates.py | 16 +---
> elbepack/virtapt.py | 146 +++++++++++++++++--------------------
> 2 files changed, 71 insertions(+), 91 deletions(-)
>
> diff --git a/elbepack/commands/check_updates.py b/elbepack/commands/check_updates.py
> index 08c006bb6..4e28bacc8 100644
> --- a/elbepack/commands/check_updates.py
> +++ b/elbepack/commands/check_updates.py
> @@ -16,6 +16,7 @@ from elbepack import virtapt
> from elbepack.validate import validate_xml
> from elbepack.xmldefaults import ElbeDefaults
> from elbepack.shellhelper import system
> +from elbepack.elbexml import ElbeXML
>
>
> def run_command(argv):
> @@ -51,27 +52,18 @@ def run_command(argv):
>
> print("checking %s" % args[0])
>
> - xml = etree(args[0])
> + xml = ElbeXML(args[0])
>
> - if xml.has("project/buildtype"):
> - buildtype = xml.text("/project/buildtype")
> - else:
> - buildtype = "nodefaults"
> -
> - defs = ElbeDefaults(buildtype)
> -
> - arch = xml.text("project/buildimage/arch", default=defs, key="arch")
> - suite = xml.text("project/suite")
>
> apt_sources = xml.text("sources_list").replace("10.0.2.2", "localhost")
> apt_prefs = xml.text("apt_prefs")
>
> fullp = xml.node("fullpkgs")
>
> - v = virtapt.VirtApt(arch, suite, apt_sources, apt_prefs)
> + v = virtapt.VirtApt(xml)
>
> d = virtapt.apt_pkg.DepCache(v.cache)
> - d.read_pinfile(v.projectpath + "/etc/apt/preferences")
> + d.read_pinfile(v.basefs.path + "/etc/apt/preferences")
>
> for p in fullp:
> pname = p.et.text
> diff --git a/elbepack/virtapt.py b/elbepack/virtapt.py
> index e064b7328..8e50ebe1a 100644
> --- a/elbepack/virtapt.py
> +++ b/elbepack/virtapt.py
> @@ -11,7 +11,6 @@ from __future__ import print_function
> import os
> import sys
>
> -from tempfile import mkdtemp
> from multiprocessing.managers import BaseManager
>
> from elbepack.shellhelper import system
> @@ -24,6 +23,9 @@ import apt_pkg
>
>
> from elbepack.shellhelper import CommandError, system
> +from elbepack.filesystem import TmpdirFilesystem
> +from elbepack.xmldefaults import ElbeDefaults
> +from elbepack.rfs import create_apt_prefs
>
>
> def getdeps(pkg):
> @@ -73,37 +75,51 @@ def lookup_uri(v, d, target_pkg):
> return target_pkg, uri, hashval
>
>
> -
> class VirtApt(object):
> - def __init__(self, arch, suite, sources, prefs, keylist=None, noauth=False):
> + def __init__(self, xml):
> +
> + self.xml = xml
>
> - # pylint: disable=too-many-arguments
> + if xml.has("project/buildtype"):
> + buildtype = xml.text("/project/buildtype")
> + else:
> + buildtype = "nodefaults"
>
> - self.projectpath = mkdtemp()
> + arch = xml.text("project/buildimage/arch", key="arch")
> + suite = xml.text("project/suite")
> +
> + self.basefs = TmpdirFilesystem()
> self.initialize_dirs()
>
> - self.create_apt_sources_list(sources)
> - self.create_apt_prefs(prefs)
> + create_apt_prefs(self.xml, self.basefs)
> +
> + mirror = self.xml.create_apt_sources_list(build_sources=True, initvm=False)
> + self.basefs.write_file("etc/apt/sources.list", 0o644, mirror)
> +
> self.setup_gpg()
> - if keylist:
> - for k in keylist:
> - self.add_pubkey_url(k)
> + self.import_keys()
>
> apt_pkg.config.set("APT::Architecture", arch)
> apt_pkg.config.set("APT::Architectures", arch)
> apt_pkg.config.set("Acquire::http::Proxy::127.0.0.1", "DIRECT")
> apt_pkg.config.set("APT::Install-Recommends", "0")
> - apt_pkg.config.set("Dir::Etc", self.projectpath)
> + apt_pkg.config.set("Dir::Etc", self.basefs.fname('/'))
> + apt_pkg.config.set("Dir::Etc::Trusted",
> + self.basefs.fname('/etc/apt/trusted.gpg'))
> + apt_pkg.config.set("Dir::Etc::TrustedParts",
> + self.basefs.fname('/etc/apt/trusted.gpg.d'))
> apt_pkg.config.set("APT::Cache-Limit", "0")
> apt_pkg.config.set("APT::Cache-Start", "32505856")
> apt_pkg.config.set("APT::Cache-Grow", "2097152")
> - apt_pkg.config.set("Dir::State", os.path.join(self.projectpath, "state"))
> - apt_pkg.config.set("Dir::State::status", os.path.join(self.projectpath, "state/status"))
> - apt_pkg.config.set("Dir::Cache", os.path.join(self.projectpath, "cache"))
> - apt_pkg.config.set("Dir::Cache::archives", os.path.join(self.projectpath, "cache/archives"))
> - apt_pkg.config.set("Dir::Etc", os.path.join(self.projectpath, "etc/apt"))
> - apt_pkg.config.set("Dir::Log", os.path.join(self.projectpath, "log"))
> - if noauth:
> + apt_pkg.config.set("Dir::State", self.basefs.fname("state"))
> + apt_pkg.config.set("Dir::State::status",
> + self.basefs.fname("state/status"))
> + apt_pkg.config.set("Dir::Cache", self.basefs.fname("cache"))
> + apt_pkg.config.set("Dir::Cache::archives",
> + self.basefs.fname("cache/archives"))
> + apt_pkg.config.set("Dir::Etc", self.basefs.fname("etc/apt"))
> + apt_pkg.config.set("Dir::Log", self.basefs.fname("log"))
> + if self.xml.has('project/noauth'):
> apt_pkg.config.set("APT::Get::AllowUnauthenticated", "1")
> apt_pkg.config.set("Acquire::AllowInsecureRepositories", "1")
> else:
> @@ -128,8 +144,27 @@ class VirtApt(object):
> except BaseException as e:
> print(e)
>
> - def __del__(self):
> - system('rm -rf "%s"' % self.projectpath)
> + def add_key(self, key):
> + cmd = 'echo "%s" > %s' % (key, self.basefs.fname("tmp/key.pub"))
> + clean = 'rm -f %s' % self.basefs.fname("tmp/key.pub")
> + system(cmd)
> + system('fakeroot apt-key --keyring "%s" add "%s"' %
> + (self.basefs.fname('/etc/apt/trusted.gpg'),
> + self.basefs.fname("tmp/key.pub")))
> + system(clean)
> +
> + def import_keys(self):
> + if self.xml.has('project/mirror/url-list'):
> + # Should we use self.xml.prj.has("noauth")???
> + #
> + # If so, this is related to issue #220 -
> + # https://github.com/Linutronix/elbe/issues/220
> + #
> + # I could make a none global 'noauth' flag for mirrors
> + for url in self.xml.node('project/mirror/url-list'):
> + if url.has('raw-key'):
> + key = "\n".join(line.strip(" \t") for line in url.text('raw-key').splitlines()[1:-1])
> + self.add_key(key)
>
> def start(self):
> pass
> @@ -156,24 +191,18 @@ class VirtApt(object):
> # mode is not set correctly
> system("chmod 777 " + newdir)
>
> - def touch(self, fname):
> - if os.path.exists(fname):
> - os.utime(fname, None)
> - else:
> - fp = open(fname, "w")
> - fp.close()
> -
> def initialize_dirs(self):
> - self.mkdir_p(os.path.join(self.projectpath, "cache/archives/partial"))
> - self.mkdir_p(os.path.join(self.projectpath, "etc/apt/preferences.d"))
> - self.mkdir_p(os.path.join(self.projectpath, "etc/apt/trusted.gpg.d"))
> - self.mkdir_p(os.path.join(self.projectpath, "db"))
> - self.mkdir_p(os.path.join(self.projectpath, "log"))
> - self.mkdir_p(os.path.join(self.projectpath, "state/lists/partial"))
> - self.touch(os.path.join(self.projectpath, "state/status"))
> + self.basefs.mkdir_p("cache/archives/partial")
> + self.basefs.mkdir_p("etc/apt/preferences.d")
> + self.basefs.mkdir_p("etc/apt/trusted.gpg.d")
> + self.basefs.mkdir_p("db")
> + self.basefs.mkdir_p("log")
> + self.basefs.mkdir_p("state/lists/partial")
> + self.basefs.mkdir_p("tmp")
> + self.basefs.touch_file("state/status")
>
> def setup_gpg(self):
> - ring_path = os.path.join(self.projectpath, "etc/apt/trusted.gpg")
> + ring_path = self.basefs.fname("etc/apt/trusted.gpg")
> if not os.path.isdir("/etc/apt/trusted.gpg.d"):
> print("/etc/apt/trusted.gpg.d doesn't exist")
> print("apt-get install debian-archive-keyring may "
> @@ -183,53 +212,12 @@ class VirtApt(object):
> if os.path.exists("/etc/apt/trusted.gpg"):
> system('cp /etc/apt/trusted.gpg "%s"' % ring_path)
>
> - gpg_options = '--keyring "%s" --no-auto-check-trustdb ' \
> - '--trust-model always --no-default-keyring ' \
> - '--homedir "%s"' % (ring_path, self.projectpath)
> -
> trustkeys = os.listdir("/etc/apt/trusted.gpg.d")
> for key in trustkeys:
> - print("Import %s: " % key)
> - try:
> - system('gpg %s --import "%s"' % (
> - gpg_options,
> - os.path.join("/etc/apt/trusted.gpg.d", key)))
> - except CommandError:
> - print("adding elbe-pubkey to keyring failed")
> -
> - def add_pubkey_url(self, url):
> - ring_path = os.path.join(self.projectpath, "etc/apt/trusted.gpg")
> - tmpkey_path = os.path.join(self.projectpath, "tmpkey.gpg")
> -
> - gpg_options = '--keyring "%s" --no-auto-check-trustdb ' \
> - '--trust-model always --no-default-keyring ' \
> - '--homedir "%s"' % (ring_path, self.projectpath)
> -
> - try:
> - system('wget -O "%s" "%s"' % (tmpkey_path, url))
> - system('gpg %s --import "%s"' % (
> - gpg_options,
> - tmpkey_path))
> - finally:
> - system('rm "%s"' % tmpkey_path, allow_fail=True)
> -
> - def create_apt_sources_list(self, mirror):
> - filename = os.path.join(self.projectpath, "etc/apt/sources.list")
> -
> - if os.path.lexists(filename):
> - os.remove(filename)
> -
> - with open(filename, "w") as f:
> - f.write(mirror)
> -
> - def create_apt_prefs(self, prefs):
> - filename = os.path.join(self.projectpath, "etc/apt/preferences")
> -
> - if os.path.lexists(filename):
> - os.remove(filename)
> + system('cp "/etc/apt/trusted.gpg.d/%s" "%s"' % (
> + key,
> + ring_path + '.d'))
>
> - with open(filename, "w") as f:
> - f.write(prefs)
>
> def get_uri(self, target_pkg, incl_deps=False):
>
>
More information about the elbe-devel
mailing list