[elbe-devel] [PATCH] Debian mirror without source packages does not end in cryptic error any more.
Manuel Traut
manuel.traut at linutronix.de
Fri May 5 15:18:31 CEST 2017
Hi Philipp,
> When building a source CDROM one needs the source files from the Debian
> mirror. If it does not provide any, the build fails in a cryptic error
> message. This patch solves the problem by checking whether source files
> are available or not and aborting right at the beginning.
>
> https://github.com/Linutronix/elbe/issues/62
thx for fixing this issue!
Regards,
Manuel
> Signed-off-by: Philipp Arras <philipp.arras at linutronix.de>
> ---
> elbepack/commands/chroot.py | 4 ++--
> elbepack/commands/control.py | 4 +++-
> elbepack/commands/pkgdiff.py | 4 ++--
> elbepack/daemons/soap/esoap.py | 15 ++++++++-------
> elbepack/db.py | 12 ++++++------
> elbepack/elbeproject.py | 25 +++++++++++++++++--------
> elbepack/elbexml.py | 24 ++++++++++++++++++------
> elbepack/initvmaction.py | 10 +++++-----
> elbepack/projectmanager.py | 9 +++++----
> elbepack/soapclient.py | 4 ++--
> 10 files changed, 68 insertions(+), 43 deletions(-)
>
> diff --git a/elbepack/commands/chroot.py b/elbepack/commands/chroot.py
> index 4e72dbb..b113d44 100644
> --- a/elbepack/commands/chroot.py
> +++ b/elbepack/commands/chroot.py
> @@ -22,7 +22,7 @@ import sys
> import os
>
> from elbepack.elbeproject import ElbeProject
> -from elbepack.elbexml import ValidationError
> +from elbepack.elbexml import ValidationError, ValidationMode
>
> def run_command( argv ):
> oparser = OptionParser(usage="usage: %prog chroot [options] <builddir> [cmd]")
> @@ -44,7 +44,7 @@ def run_command( argv ):
>
> try:
> project = ElbeProject(args[0], override_buildtype=opt.buildtype,
> - skip_validate=opt.skip_validation, skip_urlcheck=True)
> + skip_validate=opt.skip_validation, url_validation=ValidationMode.NO_CHECK)
> except ValidationError as e:
> print str(e)
> print "xml validation failed. Bailing out"
> diff --git a/elbepack/commands/control.py b/elbepack/commands/control.py
> index e058e3b..1528fa4 100755
> --- a/elbepack/commands/control.py
> +++ b/elbepack/commands/control.py
> @@ -32,6 +32,8 @@ from elbepack.soapclient import ClientAction, ElbeSoapClient
> from elbepack.version import elbe_version
> from elbepack.config import cfg
>
> +from elbepack.elbexml import ValidationMode
> +
> def run_command (argv):
> oparser = OptionParser (usage="usage: elbe control [options] <command>")
>
> @@ -73,7 +75,7 @@ def run_command (argv):
> devel = OptionGroup(oparser, "options for elbe developers",
> "Caution: Don't use these options in a productive environment")
> devel.add_option( "--skip-urlcheck", action="store_true",
> - dest="skip_urlcheck", default=False,
> + dest="url_validation", default=ValidationMode.CHECK_ALL,
> help="Skip URL Check inside initvm" )
>
> devel.add_option ("--debug", action="store_true",
> diff --git a/elbepack/commands/pkgdiff.py b/elbepack/commands/pkgdiff.py
> index 0e1a588..ec14f01 100644
> --- a/elbepack/commands/pkgdiff.py
> +++ b/elbepack/commands/pkgdiff.py
> @@ -24,7 +24,7 @@ import apt_pkg
>
> from optparse import OptionParser
>
> -from elbepack.elbexml import ElbeXML
> +from elbepack.elbexml import ElbeXML, ValidationMode
>
> def run_command( argv ):
>
> @@ -42,7 +42,7 @@ def run_command( argv ):
> fix_rfs = args[1]
>
> x = os.path.join(gen_rfs, 'etc/elbe_base.xml')
> - xml = ElbeXML (x, skip_validate=True, skip_urlcheck=True)
> + xml = ElbeXML (x, skip_validate=True, url_validation=ValidationMode.NO_CHECK)
> arch = xml.text ('project/arch', key='arch')
>
> apt_pkg.init_config()
> diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py
> index cc608f4..daba6f2 100644
> --- a/elbepack/daemons/soap/esoap.py
> +++ b/elbepack/daemons/soap/esoap.py
> @@ -29,6 +29,7 @@ from elbepack.version import elbe_version
> from faults import soap_faults
>
> from elbepack.db import Project, User
> +from elbepack.elbexml import ValidationMode
> from datatypes import SoapProject, SoapFile
> from authentication import authenticated_admin, authenticated_uid
>
> @@ -100,7 +101,7 @@ class ESoap (ServiceBase):
> if (fname == "source.xml"):
> # ensure that the project cache is reloaded
> self.app.pm.close_current_project (uid)
> - self.app.pm.open_project (uid, builddir, skip_urlcheck=True)
> + self.app.pm.open_project (uid, builddir, url_validation=ValidationMode.NO_CHECK)
> self.app.pm.set_current_project_xml (uid, fn)
> return -2
>
> @@ -164,7 +165,7 @@ class ESoap (ServiceBase):
> @authenticated_uid
> @soap_faults
> def start_cdrom (self, uid, builddir):
> - self.app.pm.open_project (uid, builddir, skip_urlcheck=True)
> + self.app.pm.open_project (uid, builddir, url_validation=ValidationMode.NO_CHECK)
>
> cdrom_fname = os.path.join (builddir, "uploaded_cdrom.iso")
>
> @@ -176,7 +177,7 @@ class ESoap (ServiceBase):
> @authenticated_uid
> @soap_faults
> def append_cdrom (self, uid, builddir, data):
> - self.app.pm.open_project (uid, builddir, skip_urlcheck=True)
> + self.app.pm.open_project (uid, builddir, url_validation=ValidationMode.NO_CHECK)
>
> cdrom_fname = os.path.join (builddir, "uploaded_cdrom.iso")
>
> @@ -189,7 +190,7 @@ class ESoap (ServiceBase):
> @authenticated_uid
> @soap_faults
> def finish_cdrom (self, uid, builddir):
> - self.app.pm.open_project (uid, builddir, skip_urlcheck=True)
> + self.app.pm.open_project (uid, builddir, url_validation=ValidationMode.NO_CHECK)
> self.app.pm.set_current_project_upload_cdrom (uid)
>
> @rpc (String)
> @@ -276,18 +277,18 @@ class ESoap (ServiceBase):
> @rpc(String, String, _returns=String)
> @authenticated_uid
> @soap_faults
> - def create_project (self, uid, xml, skip_urlcheck):
> + def create_project (self, uid, xml, url_validation):
> with NamedTemporaryFile() as fp:
> fp.write (binascii.a2b_base64 (xml))
> fp.flush ()
> - prjid = self.app.pm.create_project (uid, fp.name, skip_urlcheck=skip_urlcheck)
> + prjid = self.app.pm.create_project (uid, fp.name, url_validation=url_validation)
>
> return prjid
>
> @rpc(String, _returns=String)
> @authenticated_uid
> @soap_faults
> - def new_project (self, uid, skip_urlcheck):
> + def new_project (self, uid, url_validation):
> return self.app.pm.new_project (uid)
>
> @rpc (String, Integer, _returns=String)
> diff --git a/elbepack/db.py b/elbepack/db.py
> index 288f59e..120ed5c 100644
> --- a/elbepack/db.py
> +++ b/elbepack/db.py
> @@ -41,7 +41,7 @@ from sqlalchemy.orm.exc import NoResultFound
> from sqlalchemy.exc import OperationalError
>
> from elbepack.elbeproject import ElbeProject
> -from elbepack.elbexml import (ElbeXML, ValidationError)
> +from elbepack.elbexml import (ElbeXML, ValidationError, ValidationMode)
> from elbepack.dosunix import dos2unix
>
> Base = declarative_base ()
> @@ -278,7 +278,7 @@ class ElbeDB(object):
> "cannot set XML file while project %s is busy" %
> builddir )
>
> - xml = ElbeXML (xml_file, skip_urlcheck=True) #ValidationError
> + xml = ElbeXML (xml_file, url_validation=ValidationMode.NO_CHECK) #ValidationError
>
> p.name = xml.text ("project/name")
> p.version = xml.text ("project/version")
> @@ -419,7 +419,7 @@ class ElbeDB(object):
> project.xml = xml_str
>
>
> - def load_project (self, builddir, logpath = None, skip_urlcheck=False):
> + def load_project (self, builddir, logpath = None, url_validation=ValidationMode.CHECK_ALL):
>
> # pass exceptions if hook-scripts can't be loaded (they're optional)
> postbuild_file = None
> @@ -460,7 +460,7 @@ class ElbeDB(object):
> presh_file=presh_file,
> postsh_file=postsh_file,
> savesh_file=savesh_file,
> - skip_urlcheck=skip_urlcheck)
> + url_validation=url_validation)
> except NoResultFound:
> raise ElbeDBError(
> "project %s is not registered in the database" %
> @@ -574,7 +574,7 @@ class ElbeDB(object):
> " set_project_version: invalid status: " + p.status )
>
> xmlpath = os.path.join( builddir, "source.xml" )
> - xml = ElbeXML( xmlpath, skip_urlcheck=True )
> + xml = ElbeXML( xmlpath, url_validation=ValidationMode.NO_CHECK )
>
> if not new_version is None:
> xml.node( "/project/version" ).set_text( new_version )
> @@ -608,7 +608,7 @@ class ElbeDB(object):
> assert p.status == "busy"
>
> sourcexmlpath = os.path.join( builddir, "source.xml" )
> - sourcexml = ElbeXML( sourcexmlpath, skip_urlcheck=True )
> + sourcexml = ElbeXML( sourcexmlpath, url_validation=ValidationMode.NO_CHECK )
>
> version = sourcexml.text( "project/version" )
> if s.query( ProjectVersion ).\
> diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py
> index 73e2fad..7197b5f 100644
> --- a/elbepack/elbeproject.py
> +++ b/elbepack/elbeproject.py
> @@ -24,7 +24,7 @@ import io
>
> from elbepack.asciidoclog import ASCIIDocLog, StdoutLog
> from elbepack.shellhelper import CommandError
> -from elbepack.elbexml import ElbeXML, NoInitvmNode, ValidationError
> +from elbepack.elbexml import ElbeXML, NoInitvmNode, ValidationError, ValidationMode
> from elbepack.rfs import BuildEnv
> from elbepack.rpcaptcache import get_rpcaptcache
> from elbepack.efilesystem import TargetFs
> @@ -57,7 +57,7 @@ class AptCacheCommitError(Exception):
> class ElbeProject (object):
> def __init__ (self, builddir, xmlpath = None, logpath = None, name = None,
> override_buildtype = None, skip_validate = False,
> - skip_urlcheck = False, rpcaptcache_notifier = None,
> + url_validation = ValidationMode.CHECK_ALL, rpcaptcache_notifier = None,
> private_data = None, postbuild_file = None, presh_file = None,
> postsh_file = None, savesh_file = None):
> self.builddir = os.path.abspath(str(builddir))
> @@ -67,7 +67,7 @@ class ElbeProject (object):
> self.name = name
> self.override_buildtype = override_buildtype
> self.skip_validate = skip_validate
> - self.skip_urlcheck = skip_urlcheck
> + self.url_validation = url_validation
> self.postbuild_file = postbuild_file
> self.presh_file = presh_file
> self.postsh_file = postsh_file
> @@ -90,11 +90,11 @@ class ElbeProject (object):
> # file of the project
> if xmlpath:
> self.xml = ElbeXML( xmlpath, buildtype=override_buildtype,
> - skip_validate=skip_validate, skip_urlcheck=skip_urlcheck )
> + skip_validate=skip_validate, url_validation=url_validation )
> else:
> sourcexmlpath = os.path.join( self.builddir, "source.xml" )
> self.xml = ElbeXML( sourcexmlpath, buildtype=override_buildtype,
> - skip_validate=skip_validate, skip_urlcheck=skip_urlcheck )
> + skip_validate=skip_validate, url_validation=url_validation )
>
> self.arch = self.xml.text( "project/arch", key="arch" )
> self.codename = self.xml.text( "project/suite" )
> @@ -206,6 +206,15 @@ class ElbeProject (object):
> # Write the log header
> self.write_log_header()
>
> + # Validate Apt Sources
> + m = ValidationMode.NO_CHECK
> + if build_bin:
> + m = ValidationMode.CHECK_BINARIES
> + if build_sources:
> + m = ValidationMode.CHECK_ALL
> + self.xml.validate_apt_sources ( m , self.arch )
> +
> +
> if (self.xml.has('target/pbuilder') and not skip_pbuild):
> if not os.path.exists ( os.path.join (self.builddir, "pbuilder") ):
> self.create_pbuilder ()
> @@ -458,7 +467,7 @@ class ElbeProject (object):
>
> newxml = ElbeXML( xmlpath, buildtype=self.override_buildtype,
> skip_validate=self.skip_validate,
> - skip_urlcheck=self.skip_urlcheck )
> + url_validation=self.url_validation )
>
> # New XML file has to have the same architecture
> oldarch = self.xml.text( "project/arch", key="arch" )
> @@ -511,7 +520,7 @@ class ElbeProject (object):
> source = self.xml
> try:
> initxml = ElbeXML( "/var/cache/elbe/source.xml",
> - skip_validate=self.skip_validate, skip_urlcheck=True )
> + skip_validate=self.skip_validate, url_validation=ValidationMode.NO_CHECK )
> self.xml.get_initvmnode_from( initxml )
> except ValidationError as e:
> self.log.printo( "/var/cache/elbe/source.xml validation failed" )
> @@ -528,7 +537,7 @@ class ElbeProject (object):
> source = ElbeXML( sourcepath,
> buildtype=self.override_buildtype,
> skip_validate=self.skip_validate,
> - skip_urlcheck=self.skip_urlcheck )
> + url_validation=self.url_validation )
>
> self.xml.get_debootstrappkgs_from( source )
> try:
> diff --git a/elbepack/elbexml.py b/elbepack/elbexml.py
> index 5e6f889..36283a4 100644
> --- a/elbepack/elbexml.py
> +++ b/elbepack/elbexml.py
> @@ -49,8 +49,13 @@ class ValidationError(Exception):
> class NoInitvmNode(Exception):
> pass
>
> +class ValidationMode():
> + NO_CHECK = True
> + CHECK_BINARIES = 2
> + CHECK_ALL = False
> +
> class ElbeXML(object):
> - def __init__(self, fname, buildtype=None, skip_validate=False, skip_urlcheck=False):
> + def __init__(self, fname, buildtype=None, skip_validate=False, url_validation=ValidationMode.NO_CHECK):
> if not skip_validate:
> validation = validate_xml (fname)
> if len (validation) != 0:
> @@ -68,8 +73,8 @@ class ElbeXML(object):
> buildtype = "nodefaults"
> self.defs = ElbeDefaults(buildtype)
>
> - if not skip_validate and not skip_urlcheck:
> - self.validate_apt_sources ()
> + if not skip_validate:
> + self.validate_apt_sources (url_validation, buildtype)
>
>
> def text(self, txt, key=None):
> @@ -142,7 +147,7 @@ class ElbeXML(object):
>
> return mirror.replace("LOCALMACHINE", "10.0.2.2")
>
> - def validate_apt_sources (self):
> + def validate_apt_sources (self, url_validation, buildtype):
> slist = self.create_apt_sources_list ()
> sources_lines = slist.split ('\n')
>
> @@ -158,9 +163,16 @@ class ElbeXML(object):
> elif l.startswith ("deb ") or l.startswith ("deb-src "):
> lsplit = l.split (" ")
> if lsplit[2].endswith('/'):
> - urls.append ("%s/%sRelease" % (lsplit[1], lsplit[2]))
> + s = "%s/%s" % (lsplit[1], lsplit[2])
> else:
> - urls.append ("%s/dists/%s/Release" % (lsplit[1], lsplit[2]))
> + s = "%s/dists/%s/" % (lsplit[1], lsplit[2])
> +
> + urls.append(s + "Release")
> + if url_validation == ValidationMode.CHECK_ALL:
> + urls.append(s + lsplit[3] + "/source/Release")
> + urls.append(s + lsplit[3] + "/binary-%s/Release" % buildtype)
> + elif url_validation == ValidationMode.CHECK_BINARIES:
> + urls.append(s + lsplit[3] + "/binary-%s/Release" % buildtype)
>
> if not self.prj:
> return
> diff --git a/elbepack/initvmaction.py b/elbepack/initvmaction.py
> index bbbd76c..ed062b5 100644
> --- a/elbepack/initvmaction.py
> +++ b/elbepack/initvmaction.py
> @@ -25,7 +25,7 @@ from elbepack.treeutils import etree
> from elbepack.directories import elbe_exe
> from elbepack.shellhelper import CommandError, system, command_out_stderr
> from elbepack.filesystem import wdfs, TmpdirFilesystem, Filesystem
> -from elbepack.elbexml import ElbeXML, ValidationError
> +from elbepack.elbexml import ElbeXML, ValidationError, ValidationMode
>
> from tempfile import NamedTemporaryFile
>
> @@ -273,7 +273,7 @@ class CreateAction(InitVMAction):
> sys.exit (20)
>
> try:
> - exml = ElbeXML (tmp.fname ('source.xml'), skip_urlcheck=True)
> + exml = ElbeXML (tmp.fname ('source.xml'), url_validation=ValidationMode.NO_CHECK)
> except ValidationError as e:
> print ('Iso image does contain a source.xml file.', file=sys.stderr)
> print ('But that xml does not validate correctly', file=sys.stderr)
> @@ -477,7 +477,7 @@ class SubmitAction(InitVMAction):
> if args[0].endswith ('.xml'):
> # We have an xml file, use that for elbe init
> xmlfile = args[0]
> - skip_urlcheck = ''
> + url_validation = ''
> elif args[0].endswith ('.iso'):
> # We have an iso image, extract xml from there.
> tmp = TmpdirFilesystem ()
> @@ -493,7 +493,7 @@ class SubmitAction(InitVMAction):
> sys.exit (20)
>
> try:
> - exml = ElbeXML (tmp.fname ('source.xml'), skip_urlcheck=True)
> + exml = ElbeXML (tmp.fname ('source.xml'), url_validation=ValidationMode.NO_CHECK)
> except ValidationError as e:
> print ('Iso image does contain a source.xml file.', file=sys.stderr)
> print ('But that xml does not validate correctly', file=sys.stderr)
> @@ -505,7 +505,7 @@ class SubmitAction(InitVMAction):
> print ('Image was generated using Elbe Version %s' % exml.get_elbe_version ())
>
> xmlfile = tmp.fname ('source.xml')
> - skip_urlcheck = '--skip-urlcheck'
> + url_validation = '--skip-urlcheck'
> cdrom = args[0]
> else:
> print ('Unknown file ending (use either xml or iso)', file=sys.stderr)
> diff --git a/elbepack/projectmanager.py b/elbepack/projectmanager.py
> index 6b17fc9..05cc040 100644
> --- a/elbepack/projectmanager.py
> +++ b/elbepack/projectmanager.py
> @@ -34,6 +34,7 @@ from elbepack.asyncworker import SaveVersionJob, CheckoutVersionJob
> from elbepack.asyncworker import APTUpdUpgrJob, BuildSysrootJob
> from elbepack.asyncworker import PdebuildJob, CreatePbuilderJob
> from elbepack.asyncworker import BuildChrootTarJob
> +from elbepack.elbexml import ValidationMode
>
> class ProjectManagerError(Exception):
> def __init__ (self, message):
> @@ -83,7 +84,7 @@ class ProjectManager(object):
> self.db.create_project( builddir, owner_id=userid )
> return builddir
>
> - def create_project (self, userid, xml_file, skip_urlcheck=False):
> + def create_project (self, userid, xml_file, url_validation=ValidationMode.CHECK_ALL):
> subdir = str(uuid4())
> builddir = path.join( self.basepath, subdir )
>
> @@ -102,14 +103,14 @@ class ProjectManager(object):
>
> # Open the new project
> logpath = path.join( builddir, "log.txt" )
> - ep = self.db.load_project( builddir, logpath, skip_urlcheck=skip_urlcheck )
> + ep = self.db.load_project( builddir, logpath, url_validation=url_validation )
>
> self.userid2project[ userid ] = ep
> self.builddir2userid[ builddir ] = userid
>
> return builddir
>
> - def open_project (self, userid, builddir, skip_urlcheck=False):
> + def open_project (self, userid, builddir, url_validation=ValidationMode.CHECK_ALL):
> self._check_project_permission( userid, builddir )
>
> with self.lock:
> @@ -129,7 +130,7 @@ class ProjectManager(object):
>
> # Load project from the database
> logpath = path.join( builddir, "log.txt" )
> - ep = self.db.load_project( builddir, logpath, skip_urlcheck=skip_urlcheck )
> + ep = self.db.load_project( builddir, logpath, url_validation=url_validation )
>
> # Add project to our dictionaries
> self.userid2project[ userid ] = ep
> diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
> index bc38a61..1a98f91 100644
> --- a/elbepack/soapclient.py
> +++ b/elbepack/soapclient.py
> @@ -30,7 +30,7 @@ import sys
> import os
>
> from elbepack.filesystem import Filesystem
> -from elbepack.elbexml import ElbeXML
> +from elbepack.elbexml import ElbeXML, ValidationMode
>
> def set_suds_debug(debug):
> import logging
> @@ -221,7 +221,7 @@ class SetXmlAction(ClientAction):
> filename = args[1]
>
> try:
> - x = ElbeXML (filename, skip_validate=True, skip_urlcheck=True)
> + x = ElbeXML (filename, skip_validate=True, url_validation=ValidationMode.NO_CHECK)
> except IOError as e:
> print ("%s is not a valid elbe xml file" % filename)
> sys.exit (20)
> --
> 2.1.4
>
>
> _______________________________________________
> elbe-devel mailing list
> elbe-devel at linutronix.de
> https://lists.linutronix.de/mailman/listinfo/elbe-devel
More information about the elbe-devel
mailing list