[elbe-devel] [PATCH v2 5/6] pkgutils: cleanup non-virtapt fallback
Torben Hohn
torben.hohn at linutronix.de
Mon Oct 15 13:30:52 CEST 2018
On Fri, Oct 12, 2018 at 05:22:59PM +0200, Manuel Traut wrote:
> On Wed, Sep 26, 2018 at 12:13:51PM +0200, Torben Hohn wrote:
> > since elbe uses elbepack.debinstaller outside of the initvm now,
> > the non-virtap fallback code it not needed anymore.
> >
> > remove it and make pkgutils.py pylint clean.
> > Also rename the, now moved to debinstaller, NoKinitrdException
> > to NoPackageException.
>
> please add the removals to the previous patch, i'd expected them there..
>
> Add an extra patch for pylint and autopep8 cleanup of this file.
> The cleanup doesnt necesarily be part of this queue. But it's ok if it's in.
i dont find the cleanup in the diff.
it probably got clean, because i removed the code.
>
> Thanks,
>
> Manu
>
> > Signed-off-by: Torben Hohn <torben.hohn at linutronix.de>
> > ---
> > elbepack/pkgutils.py | 162 ++++++++-------------------------------------------
> > 1 file changed, 23 insertions(+), 139 deletions(-)
> >
> > diff --git a/elbepack/pkgutils.py b/elbepack/pkgutils.py
> > index 08b6003d..1a619d6e 100644
> > --- a/elbepack/pkgutils.py
> > +++ b/elbepack/pkgutils.py
> > @@ -7,48 +7,17 @@
> >
> > from __future__ import print_function
> >
> > -# different module names in python 2 and 3
> > -try:
> > - import urllib.request
> > -
> > - # when running inside pylint this import fails
> > - # disable no-member here
> > - #
> > - # pylint: disable=no-member
> > -
> > - urlopen = urllib.request.urlopen
> > -except ImportError:
> > - import urllib2
> > - urlopen = urllib2.urlopen
> > -
> > import os
> > -import hashlib
> >
> > from tempfile import mkdtemp
> >
> > -from pkg_resources import parse_version as V
> > +from apt_pkg import TagFile
> > from elbepack.shellhelper import CommandError, system
> > +from elbepack.virtapt import get_virtaptcache
> > +from elbepack.hashes import validate_sha256, HashValidationFailed
> >
> > -try:
> > - # this can not be grouped properly
> > - # because we can fallback of apt_pkg is not available
> > - #
> > - # pylint: disable=ungrouped-imports
> > -
> > - from apt_pkg import TagFile
> > - from elbepack.virtapt import get_virtaptcache
> > - virtapt_imported = True
> > -except ImportError as e:
> > - print(e)
> > - print("WARNING - python-apt not available:")
> > - print("If there are multiple versions of elbe-bootstrap packages on the "
> > - "mirror(s) elbe selects the first package it has found.")
> > - print("There is no guarantee that the latest package is used.")
> > - print("To ensure this, the python-apt package needs to be installed.")
> > - virtapt_imported = False
> > -
> > -
> > -
> > +class NoPackageException(Exception):
> > + pass
> >
> > def get_sources_list(prj, defs):
> >
> > @@ -93,77 +62,6 @@ def get_key_list(prj):
> > return retval
> >
> >
> > -def get_initrd_pkg(prj, defs):
> > - initrdname = prj.text("buildimage/kinitrd", default=defs, key="kinitrd")
> > -
> > - return initrdname
> > -
> > -
> > -def get_url(arch, suite, target_pkg, mirror, comp='main'):
> > -
> > - # pylint: disable=too-many-locals
> > -
> > - try:
> > - pack_url = "%s/dists/%s/%s/binary-%s/Packages" % (
> > - mirror.replace("LOCALMACHINE", "localhost"), suite, comp, arch)
> > - packages = urllib2.urlopen(pack_url, None, 10)
> > -
> > - packages = packages.readlines()
> > - packages = [x for x in packages if x.startswith("Filename")]
> > - packages = [x for x in packages if x.find(target_pkg) != -1]
> > -
> > - # detect package with latest version number
> > - latest_version_str = '0+deb0u0+deb8'
> > - latest_version_pos = 0
> > - cnt = 0
> > - for x in packages:
> > - # extract version from path/name_version_arch
> > - version = x.split('_')[1]
> > - subcount = 0
> > - # iterate over all parts of the version seperated by '+'
> > - # this is enough for elbe-bootstrap package, however '~', etc.
> > - # should be considered for official debian packages..
> > - for subv in version.split('+'):
> > - try:
> > - if V(subv) >= V(latest_version_str.split('+')[subcount]):
> > - subcount = subcount + 1
> > - else:
> > - break
> > - # current version has more parts then the reference version
> > - except IndexError:
> > - subcount = subcount + 1
> > - # if iteration over all parts of the version string suceeded,
> > - # a new latest_version is detected
> > - if subcount == len(version.split('+')):
> > - latest_version_pos = cnt
> > - latest_version_str = version
> > - cnt = cnt + 1
> > -
> > - urla = packages[latest_version_pos].split()
> > - url = "%s/%s" % (mirror.replace("LOCALMACHINE", "localhost"), urla[1])
> > - except IOError:
> > - url = ""
> > - except IndexError:
> > - url = ""
> > -
> > - return url
> > -
> > -
> > -def get_uri_nonvirtapt(apt_sources, target_pkg, arch):
> > - for apts in apt_sources.splitlines():
> > - apts_split = apts.strip().split(' ')
> > - if apts_split[0] != 'deb':
> > - continue
> > -
> > - for comp in apts_split[2:]:
> > - pkg = get_url(arch, apts_split[2], target_pkg, apts_split[1], comp)
> > -
> > - if pkg:
> > - return [(target_pkg, pkg, "")]
> > -
> > - return [(target_pkg, "nonexistent://" + target_pkg, "")]
> > -
> > -
> > def get_uri(prj, defs, arch, target_pkg, incl_deps=False):
> > if arch == "default":
> > arch = prj.text("buildimage/arch", default=defs, key="arch")
> > @@ -172,26 +70,15 @@ def get_uri(prj, defs, arch, target_pkg, incl_deps=False):
> > apt_sources = get_sources_list(prj, defs)
> > apt_keys = get_key_list(prj)
> >
> > - if virtapt_imported:
> > - try:
> > - if arch == "default":
> > - arch = prj.text("buildimage/arch", default=defs, key="arch")
> > - suite = prj.text("suite")
> > - v = get_virtaptcache(arch, suite, apt_sources, "", apt_keys)
> > - except Exception as e:
> > - print("python-apt failed, using fallback code: %s" % e)
> > - return get_uri_nonvirtapt(apt_sources, target_pkg, arch)
> > -
> > - ret = v.get_uri(target_pkg, incl_deps)
> > - return ret
> > -
> > - return get_uri_nonvirtapt(apt_sources, target_pkg, arch)
> > + if arch == "default":
> > + arch = prj.text("buildimage/arch", default=defs, key="arch")
> > + suite = prj.text("suite")
> > + v = get_virtaptcache(arch, suite, apt_sources, "", apt_keys)
> >
> > + ret = v.get_uri(target_pkg, incl_deps)
> > + return ret
> >
> > def get_dsc_size(fname):
> > - if not virtapt_imported:
> > - return 0
> > -
> > tf = TagFile(fname)
> >
> > sz = os.path.getsize(fname)
> > @@ -204,7 +91,6 @@ def get_dsc_size(fname):
> >
> > return sz
> >
> > -
> > def download_pkg(prj,
> > target_dir,
> > defs,
> > @@ -220,14 +106,14 @@ def download_pkg(prj,
> > try:
> > urilist = get_uri(prj, defs, arch, package, incl_deps)
> > except KeyError:
> > - raise NoKinitrdException('no package %s available' % package)
> > + raise NoPackageException('no package %s available' % package)
> > except SystemError:
> > - raise NoKinitrdException('a configured mirror is not reachable')
> > + raise NoPackageException('a configured mirror is not reachable')
> > except CommandError:
> > - raise NoKinitrdException("couldn't download package %s" % package)
> > + raise NoPackageException("couldn't download package %s" % package)
> >
> > if not urilist:
> > - raise NoKinitrdException("couldn't download package %s" % package)
> > + raise NoPackageException("couldn't download package %s" % package)
> >
> > for u in urilist:
> > sha256 = u[2]
> > @@ -240,19 +126,17 @@ def download_pkg(prj,
> > elif uri.startswith("http://") or uri.startswith("ftp://"):
> > system('wget -O "%s" "%s"' % (dest, uri))
> > else:
> > - raise NoKinitrdException('could not retreive %s' % uri)
> > + raise NoPackageException('could not retreive %s' % uri)
> > except CommandError:
> > - raise NoKinitrdException("couldn't download package %s" % package)
> > + raise NoPackageException("couldn't download package %s" % package)
> >
> > if sha256:
> > - m = hashlib.sha256()
> > - with open(dest, "rb") as f:
> > - buf = f.read(65536)
> > - while len(buf) > 0:
> > - m.update(buf)
> > - buf = f.read(65536)
> > - if m.hexdigest() != sha256:
> > - raise NoKinitrdException('%s failed to verify !!!' % package)
> > + try:
> > + validate_sha256(dest, sha256)
> > + except HashValidationFailed as e:
> > + raise NoPackageException('%s failed to verify: %s' %
> > + package,
> > + e.message)
> > else:
> > if log:
> > log.printo("WARNING: Using untrusted %s package" % package)
> > --
> > 2.11.0
> >
> >
> > _______________________________________________
> > elbe-devel mailing list
> > elbe-devel at linutronix.de
> > https://lists.linutronix.de/mailman/listinfo/elbe-devel
>
> _______________________________________________
> elbe-devel mailing list
> elbe-devel at linutronix.de
> https://lists.linutronix.de/mailman/listinfo/elbe-devel
--
Torben Hohn
Linutronix GmbH | Bahnhofstrasse 3 | D-88690 Uhldingen-Mühlhofen
Phone: +49 7556 25 999 18; Fax.: +49 7556 25 999 99
Hinweise zum Datenschutz finden Sie hier (Informations on data privacy
can be found here): https://linutronix.de/kontakt/Datenschutz.php
Linutronix GmbH | Firmensitz (Registered Office): Uhldingen-Mühlhofen |
Registergericht (Registration Court): Amtsgericht Freiburg i.Br., HRB700
806 | Geschäftsführer (Managing Directors): Heinz Egger, Thomas Gleixner
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: not available
URL: <http://lists.linutronix.de/pipermail/elbe-devel/attachments/20181015/20845711/attachment.sig>
More information about the elbe-devel
mailing list