[elbe-devel] [PATCH v2 05/24] pkgutils: rework copy_kinitrd function
Torben Hohn
torben.hohn at linutronix.de
Thu Feb 8 16:40:08 CET 2018
On Thu, Feb 08, 2018 at 02:16:51PM +0100, Manuel Traut wrote:
> copy_kinitrd implemented download and extraction of debian packages.
> To improve readability and reusability of the code, split out the
> download and extraction of a debian package into seperate functions.
>
> Remove code-duplication by introducing a target_dir and dest variable.
>
> Also add optional support for asciidoclog to log into a file instead of
> stdout.
>
> Add the posibility to download and extract a specified pacakage
> including all its runtime dependencies. (This only works if python-apt
> is available)
>
> Signed-off-by: Manuel Traut <manut at linutronix.de>
Reviewed-by: Torben Hohn <torben.hohn at linutronix.de>
follow-up patches...
> ---
> elbepack/pkgutils.py | 196 ++++++++++++++++++++++++++++-----------------------
> 1 file changed, 107 insertions(+), 89 deletions(-)
>
> diff --git a/elbepack/pkgutils.py b/elbepack/pkgutils.py
> index 65ddc4b7..55f80f94 100644
> --- a/elbepack/pkgutils.py
> +++ b/elbepack/pkgutils.py
> @@ -132,13 +132,35 @@ def get_uri_nonvirtapt(apt_sources, target_pkg, arch):
> if pkg:
> return "", pkg
>
> +def getdeps(pkg):
> + for dd in pkg.depends_list.get("Depends", []):
> + for d in dd:
> + yield d.target_pkg.name
>
> -def get_initrd_uri(prj, defs, arch):
> - target_pkg = get_initrd_pkg(prj, defs)
> - return get_uri(prj, defs, arch, target_pkg)
> +def lookup_uri(v, d, target_pkg):
> +
> + pkg = v.cache[target_pkg]
> +
> + c = d.get_candidate_ver(pkg)
> +
> + x = v.source.find_index(c.file_list[0][0])
> +
> + r = virtapt.apt_pkg.PackageRecords(v.cache)
> + r.lookup(c.file_list[0])
> + uri = x.archive_uri(r.filename)
> +
> + if not x.is_trusted:
> + return target_pkg, uri, ""
> +
> + # TODO remove r.sha256_hash path as soon as initvm is stretch or later
> + try:
> + hashval = r.sha256_hash
> + except DeprecationWarning:
> + hashval = str(r.hashes.find('SHA256')).split(':')[1]
>
> + return target_pkg, uri, hashval
>
> -def get_uri(prj, defs, arch, target_pkg):
> +def get_uri(prj, defs, arch, target_pkg, incl_deps=False):
> if arch == "default":
> arch = prj.text("buildimage/arch", default=defs, key="arch")
> suite = prj.text("suite")
> @@ -153,24 +175,27 @@ def get_uri(prj, defs, arch, target_pkg):
> return get_uri_nonvirtapt(apt_sources, target_pkg, arch)
>
> d = virtapt.apt_pkg.DepCache(v.cache)
> - pkg = v.cache[target_pkg]
>
> - c = d.get_candidate_ver(pkg)
> - x = v.source.find_index(c.file_list[0][0])
> + if not incl_deps:
> + return [lookup_uri(v, d, target_pkg)]
> +
> + if incl_deps:
> + deps = [lookup_uri(v, d, target_pkg)]
> + togo = [target_pkg]
> + while len(togo):
> + pp = togo.pop()
> + pkg= v.cache[pp]
> + c = d.get_candidate_ver(pkg)
> + for p in getdeps(c):
> + if len([y for y in deps if y[0] == p]):
> + continue
> + if p != target_pkg and p == pp:
> + continue
> + deps.append(lookup_uri(v, d, p))
> + togo.append(p)
> +
> + return deps
>
> - r = virtapt.apt_pkg.PackageRecords(v.cache)
> - r.lookup(c.file_list[0])
> - uri = x.archive_uri(r.filename)
> -
> - if not x.is_trusted:
> - return "", uri
> -
> - # TODO remove r.sha256_hash path as soon as initvm is stretch or later
> - try:
> - return r.sha256_hash, uri
> - except DeprecationWarning:
> - hashval = r.hashes.find('SHA256')
> - return hashval, uri
> else:
> return get_uri_nonvirtapt(apt_sources, target_pkg, arch)
>
> @@ -194,107 +219,100 @@ def get_dsc_size(fname):
> return sz
>
>
> -def copy_kinitrd(prj, target_dir, defs, arch="default"):
> +def download_pkg(prj,
> + target_dir,
> + defs,
> + package,
> + arch="default",
> + incl_deps=False,
> + log=None):
> +
> try:
> - sha256, uri = get_initrd_uri(prj, defs, arch)
> + urilist = get_uri(prj, defs, arch, package, incl_deps)
> except KeyError:
> - raise NoKinitrdException('no elbe-bootstrap package available')
> + raise NoKinitrdException('no package %s available' % package)
> return
> except SystemError:
> raise NoKinitrdException('a configured mirror is not reachable')
> return
> except CommandError as e:
> - raise NoKinitrdException("couldn't download elbe-bootstrap package")
> - return
> + raise NoKinitrdException("couldn't download package %s" % package)
>
> - try:
> - tmpdir = mkdtemp()
> + for u in urilist:
> + sha256 = u[2]
> + uri = u[1]
> + dest = os.path.join(target_dir, "%s.deb" % u[0])
>
> try:
> if uri.startswith("file://"):
> - system('cp "%s" "%s"' %
> - (uri[len("file://"):], os.path.join(tmpdir, "pkg.deb")))
> - elif uri.startswith("http://"):
> - system('wget -O "%s" "%s"' %
> - (os.path.join(tmpdir, "pkg.deb"), uri))
> - elif uri.startswith("ftp://"):
> - system('wget -O "%s" "%s"' %
> - (os.path.join(tmpdir, "pkg.deb"), uri))
> + system('cp "%s" "%s"' % (uri[len("file://"):], dest))
> + elif uri.startswith("http://") or uri.startswith("ftp://"):
> + system('wget -O "%s" "%s"' % (dest, uri))
> else:
> - raise NoKinitrdException('no elbe-bootstrap package available')
> + raise NoKinitrdException('could not retreive %s' % uri)
> except CommandError as e:
> - raise NoKinitrdException(
> - "couldn't download elbe-bootstrap package")
> - return
> + raise NoKinitrdException("couldn't download package %s" % package)
>
> if len(sha256) > 0:
> m = hashlib.sha256()
> - with open(os.path.join(tmpdir, "pkg.deb"), "rb") as f:
> + with open(dest, "rb") as f:
> buf = f.read(65536)
> while len(buf) > 0:
> m.update(buf)
> buf = f.read(65536)
> -
> if m.hexdigest() != sha256:
> - raise NoKinitrdException('elbe-bootstrap failed to verify !!!')
> + raise NoKinitrdException('%s failed to verify !!!' % package)
> else:
> - print("-----------------------------------------------------")
> - print("WARNING:")
> - print("Using untrusted elbe-bootstrap")
> - print("-----------------------------------------------------")
> + if log:
> + log.printo("WARNING: Using untrusted %s package" % package)
> + else:
> + print("-----------------------------------------------------")
> + print("WARNING:")
> + print("Using untrusted %s package" % package)
> + print("-----------------------------------------------------")
> +
> + return [y[0] for y in urilist]
> +
>
> +def extract_pkg(prj, target_dir, defs, package, arch="default",
> + incl_deps=False, log=None):
> +
> + pkgs = download_pkg(prj, target_dir, defs, package, arch, incl_deps, log)
> +
> + for package in pkgs:
> + ppath = os.path.join(target_dir, "%s.deb" % package)
> try:
> - system('dpkg -x "%s" "%s"' %
> - (os.path.join(tmpdir, "pkg.deb"), tmpdir))
> + system('dpkg -x "%s" "%s"' % (ppath, target_dir))
> except CommandError:
> try:
> # dpkg did not work, try falling back to ar and tar
> - system('ar p "%s" data.tar.gz | tar xz -C "%s"' %
> - (os.path.join(tmpdir, "pkg.deb"), tmpdir))
> + system('ar p "%s" data.tar.gz | tar xz -C "%s"' % (ppath,
> + target_dir))
> except CommandError:
> - system('ar p "%s" data.tar.xz | tar xJ -C "%s"' %
> - (os.path.join(tmpdir, "pkg.deb"), tmpdir))
> + system('ar p "%s" data.tar.xz | tar xJ -C "%s"' % (ppath,
> + target_dir))
> + system('rm -f "%s"' % ppath)
> +
> +
> +def copy_kinitrd(prj, target_dir, defs, arch="default"):
> +
> + target_pkg = get_initrd_pkg(prj, defs)
> +
> + try:
> + tmpdir = mkdtemp()
> + extract_pkg(prj, tmpdir, defs, target_pkg, arch)
>
> # copy is done twice, because paths in elbe-bootstarp_1.0 and 0.9
> # differ
> + initrd = os.path.join(tmpdir, 'var', 'lib', 'elbe', 'initrd')
> if prj.has("mirror/cdrom"):
> - system(
> - 'cp "%s" "%s"' %
> - (os.path.join(
> - tmpdir,
> - 'var',
> - 'lib',
> - 'elbe',
> - 'initrd',
> - 'initrd-cdrom.gz'),
> - os.path.join(
> - target_dir,
> - "initrd.gz")))
> + system('cp "%s" "%s"' % (os.path.join(initrd, 'initrd-cdrom.gz'),
> + os.path.join(target_dir, "initrd.gz")))
> else:
> - system(
> - 'cp "%s" "%s"' %
> - (os.path.join(
> - tmpdir,
> - 'var',
> - 'lib',
> - 'elbe',
> - 'initrd',
> - 'initrd.gz'),
> - os.path.join(
> - target_dir,
> - "initrd.gz")))
> -
> - system(
> - 'cp "%s" "%s"' %
> - (os.path.join(
> - tmpdir,
> - 'var',
> - 'lib',
> - 'elbe',
> - 'initrd',
> - 'vmlinuz'),
> - os.path.join(
> - target_dir,
> - "vmlinuz")))
> + system('cp "%s" "%s"' % (os.path.join(initrd, 'initrd.gz'),
> + os.path.join(target_dir, "initrd.gz")))
> +
> + system('cp "%s" "%s"' % (os.path.join(initrd, 'vmlinuz'),
> + os.path.join(target_dir, "vmlinuz")))
> finally:
> system('rm -rf "%s"' % tmpdir)
> --
> 2.15.1
>
--
Mit freundlichen Grüßen
Torben Hohn
Linutronix GmbH
Standort: Bremen
Phone: +49 7556 25 999 18; Fax.: +49 7556 25 999 99
Firmensitz / Registered Office: D-88690 Uhldingen, Bahnhofstr. 3
Registergericht / Local District Court: Amtsgericht Freiburg i. Br.; HRB
Nr. / Trade register no.: 700 806
Geschäftsführer / Managing Directors: Heinz Egger, Thomas Gleixner
Eine Bitte von uns: Sollten Sie diese E-Mail irrtümlich erhalten haben,
benachrichtigen Sie uns in diesem Falle bitte sobald wie es Ihnen
möglich ist, durch Antwort-Mail. Vielen Dank!
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: not available
URL: <http://lists.linutronix.de/pipermail/elbe-devel/attachments/20180208/70f11c09/attachment-0001.sig>
More information about the elbe-devel
mailing list