[elbe-devel] [PATCH v2 5/6] pkgutils: cleanup non-virtapt fallback
Manuel Traut
manut at linutronix.de
Fri Oct 12 17:22:59 CEST 2018
On Wed, Sep 26, 2018 at 12:13:51PM +0200, Torben Hohn wrote:
> since elbe uses elbepack.debinstaller outside of the initvm now,
> the non-virtap fallback code it not needed anymore.
>
> remove it and make pkgutils.py pylint clean.
> Also rename the, now moved to debinstaller, NoKinitrdException
> to NoPackageException.
please add the removals to the previous patch, i'd expected them there..
Add an extra patch for pylint and autopep8 cleanup of this file.
The cleanup doesnt necesarily be part of this queue. But it's ok if it's in.
Thanks,
Manu
> Signed-off-by: Torben Hohn <torben.hohn at linutronix.de>
> ---
> elbepack/pkgutils.py | 162 ++++++++-------------------------------------------
> 1 file changed, 23 insertions(+), 139 deletions(-)
>
> diff --git a/elbepack/pkgutils.py b/elbepack/pkgutils.py
> index 08b6003d..1a619d6e 100644
> --- a/elbepack/pkgutils.py
> +++ b/elbepack/pkgutils.py
> @@ -7,48 +7,17 @@
>
> from __future__ import print_function
>
> -# different module names in python 2 and 3
> -try:
> - import urllib.request
> -
> - # when running inside pylint this import fails
> - # disable no-member here
> - #
> - # pylint: disable=no-member
> -
> - urlopen = urllib.request.urlopen
> -except ImportError:
> - import urllib2
> - urlopen = urllib2.urlopen
> -
> import os
> -import hashlib
>
> from tempfile import mkdtemp
>
> -from pkg_resources import parse_version as V
> +from apt_pkg import TagFile
> from elbepack.shellhelper import CommandError, system
> +from elbepack.virtapt import get_virtaptcache
> +from elbepack.hashes import validate_sha256, HashValidationFailed
>
> -try:
> - # this can not be grouped properly
> - # because we can fallback of apt_pkg is not available
> - #
> - # pylint: disable=ungrouped-imports
> -
> - from apt_pkg import TagFile
> - from elbepack.virtapt import get_virtaptcache
> - virtapt_imported = True
> -except ImportError as e:
> - print(e)
> - print("WARNING - python-apt not available:")
> - print("If there are multiple versions of elbe-bootstrap packages on the "
> - "mirror(s) elbe selects the first package it has found.")
> - print("There is no guarantee that the latest package is used.")
> - print("To ensure this, the python-apt package needs to be installed.")
> - virtapt_imported = False
> -
> -
> -
> +class NoPackageException(Exception):
> + pass
>
> def get_sources_list(prj, defs):
>
> @@ -93,77 +62,6 @@ def get_key_list(prj):
> return retval
>
>
> -def get_initrd_pkg(prj, defs):
> - initrdname = prj.text("buildimage/kinitrd", default=defs, key="kinitrd")
> -
> - return initrdname
> -
> -
> -def get_url(arch, suite, target_pkg, mirror, comp='main'):
> -
> - # pylint: disable=too-many-locals
> -
> - try:
> - pack_url = "%s/dists/%s/%s/binary-%s/Packages" % (
> - mirror.replace("LOCALMACHINE", "localhost"), suite, comp, arch)
> - packages = urllib2.urlopen(pack_url, None, 10)
> -
> - packages = packages.readlines()
> - packages = [x for x in packages if x.startswith("Filename")]
> - packages = [x for x in packages if x.find(target_pkg) != -1]
> -
> - # detect package with latest version number
> - latest_version_str = '0+deb0u0+deb8'
> - latest_version_pos = 0
> - cnt = 0
> - for x in packages:
> - # extract version from path/name_version_arch
> - version = x.split('_')[1]
> - subcount = 0
> - # iterate over all parts of the version seperated by '+'
> - # this is enough for elbe-bootstrap package, however '~', etc.
> - # should be considered for official debian packages..
> - for subv in version.split('+'):
> - try:
> - if V(subv) >= V(latest_version_str.split('+')[subcount]):
> - subcount = subcount + 1
> - else:
> - break
> - # current version has more parts then the reference version
> - except IndexError:
> - subcount = subcount + 1
> - # if iteration over all parts of the version string suceeded,
> - # a new latest_version is detected
> - if subcount == len(version.split('+')):
> - latest_version_pos = cnt
> - latest_version_str = version
> - cnt = cnt + 1
> -
> - urla = packages[latest_version_pos].split()
> - url = "%s/%s" % (mirror.replace("LOCALMACHINE", "localhost"), urla[1])
> - except IOError:
> - url = ""
> - except IndexError:
> - url = ""
> -
> - return url
> -
> -
> -def get_uri_nonvirtapt(apt_sources, target_pkg, arch):
> - for apts in apt_sources.splitlines():
> - apts_split = apts.strip().split(' ')
> - if apts_split[0] != 'deb':
> - continue
> -
> - for comp in apts_split[2:]:
> - pkg = get_url(arch, apts_split[2], target_pkg, apts_split[1], comp)
> -
> - if pkg:
> - return [(target_pkg, pkg, "")]
> -
> - return [(target_pkg, "nonexistent://" + target_pkg, "")]
> -
> -
> def get_uri(prj, defs, arch, target_pkg, incl_deps=False):
> if arch == "default":
> arch = prj.text("buildimage/arch", default=defs, key="arch")
> @@ -172,26 +70,15 @@ def get_uri(prj, defs, arch, target_pkg, incl_deps=False):
> apt_sources = get_sources_list(prj, defs)
> apt_keys = get_key_list(prj)
>
> - if virtapt_imported:
> - try:
> - if arch == "default":
> - arch = prj.text("buildimage/arch", default=defs, key="arch")
> - suite = prj.text("suite")
> - v = get_virtaptcache(arch, suite, apt_sources, "", apt_keys)
> - except Exception as e:
> - print("python-apt failed, using fallback code: %s" % e)
> - return get_uri_nonvirtapt(apt_sources, target_pkg, arch)
> -
> - ret = v.get_uri(target_pkg, incl_deps)
> - return ret
> -
> - return get_uri_nonvirtapt(apt_sources, target_pkg, arch)
> + if arch == "default":
> + arch = prj.text("buildimage/arch", default=defs, key="arch")
> + suite = prj.text("suite")
> + v = get_virtaptcache(arch, suite, apt_sources, "", apt_keys)
>
> + ret = v.get_uri(target_pkg, incl_deps)
> + return ret
>
> def get_dsc_size(fname):
> - if not virtapt_imported:
> - return 0
> -
> tf = TagFile(fname)
>
> sz = os.path.getsize(fname)
> @@ -204,7 +91,6 @@ def get_dsc_size(fname):
>
> return sz
>
> -
> def download_pkg(prj,
> target_dir,
> defs,
> @@ -220,14 +106,14 @@ def download_pkg(prj,
> try:
> urilist = get_uri(prj, defs, arch, package, incl_deps)
> except KeyError:
> - raise NoKinitrdException('no package %s available' % package)
> + raise NoPackageException('no package %s available' % package)
> except SystemError:
> - raise NoKinitrdException('a configured mirror is not reachable')
> + raise NoPackageException('a configured mirror is not reachable')
> except CommandError:
> - raise NoKinitrdException("couldn't download package %s" % package)
> + raise NoPackageException("couldn't download package %s" % package)
>
> if not urilist:
> - raise NoKinitrdException("couldn't download package %s" % package)
> + raise NoPackageException("couldn't download package %s" % package)
>
> for u in urilist:
> sha256 = u[2]
> @@ -240,19 +126,17 @@ def download_pkg(prj,
> elif uri.startswith("http://") or uri.startswith("ftp://"):
> system('wget -O "%s" "%s"' % (dest, uri))
> else:
> - raise NoKinitrdException('could not retreive %s' % uri)
> + raise NoPackageException('could not retreive %s' % uri)
> except CommandError:
> - raise NoKinitrdException("couldn't download package %s" % package)
> + raise NoPackageException("couldn't download package %s" % package)
>
> if sha256:
> - m = hashlib.sha256()
> - with open(dest, "rb") as f:
> - buf = f.read(65536)
> - while len(buf) > 0:
> - m.update(buf)
> - buf = f.read(65536)
> - if m.hexdigest() != sha256:
> - raise NoKinitrdException('%s failed to verify !!!' % package)
> + try:
> + validate_sha256(dest, sha256)
> + except HashValidationFailed as e:
> + raise NoPackageException('%s failed to verify: %s' %
> + package,
> + e.message)
> else:
> if log:
> log.printo("WARNING: Using untrusted %s package" % package)
> --
> 2.11.0
>
>
> _______________________________________________
> elbe-devel mailing list
> elbe-devel at linutronix.de
> https://lists.linutronix.de/mailman/listinfo/elbe-devel
More information about the elbe-devel
mailing list