[elbe-devel] [PATCH] Debian mirror without source packages does not end in cryptic error any more.
Philipp Arras
philipp.arras at linutronix.de
Thu May 4 18:01:51 CEST 2017
When building a source CDROM one needs the source files from the Debian
mirror. If it does not provide any, the build fails in a cryptic error
message. This patch solves the problem by checking whether source files
are available or not and aborting right at the beginning.
https://github.com/Linutronix/elbe/issues/62
Signed-off-by: Philipp Arras <philipp.arras at linutronix.de>
---
elbepack/commands/chroot.py | 4 ++--
elbepack/commands/control.py | 4 +++-
elbepack/commands/pkgdiff.py | 4 ++--
elbepack/daemons/soap/esoap.py | 15 ++++++++-------
elbepack/db.py | 12 ++++++------
elbepack/elbeproject.py | 25 +++++++++++++++++--------
elbepack/elbexml.py | 24 ++++++++++++++++++------
elbepack/initvmaction.py | 10 +++++-----
elbepack/projectmanager.py | 9 +++++----
elbepack/soapclient.py | 4 ++--
10 files changed, 68 insertions(+), 43 deletions(-)
diff --git a/elbepack/commands/chroot.py b/elbepack/commands/chroot.py
index 4e72dbb..b113d44 100644
--- a/elbepack/commands/chroot.py
+++ b/elbepack/commands/chroot.py
@@ -22,7 +22,7 @@ import sys
import os
from elbepack.elbeproject import ElbeProject
-from elbepack.elbexml import ValidationError
+from elbepack.elbexml import ValidationError, ValidationMode
def run_command( argv ):
oparser = OptionParser(usage="usage: %prog chroot [options] <builddir> [cmd]")
@@ -44,7 +44,7 @@ def run_command( argv ):
try:
project = ElbeProject(args[0], override_buildtype=opt.buildtype,
- skip_validate=opt.skip_validation, skip_urlcheck=True)
+ skip_validate=opt.skip_validation, url_validation=ValidationMode.NO_CHECK)
except ValidationError as e:
print str(e)
print "xml validation failed. Bailing out"
diff --git a/elbepack/commands/control.py b/elbepack/commands/control.py
index e058e3b..1528fa4 100755
--- a/elbepack/commands/control.py
+++ b/elbepack/commands/control.py
@@ -32,6 +32,8 @@ from elbepack.soapclient import ClientAction, ElbeSoapClient
from elbepack.version import elbe_version
from elbepack.config import cfg
+from elbepack.elbexml import ValidationMode
+
def run_command (argv):
oparser = OptionParser (usage="usage: elbe control [options] <command>")
@@ -73,7 +75,7 @@ def run_command (argv):
devel = OptionGroup(oparser, "options for elbe developers",
"Caution: Don't use these options in a productive environment")
devel.add_option( "--skip-urlcheck", action="store_true",
- dest="skip_urlcheck", default=False,
+ dest="url_validation", default=ValidationMode.CHECK_ALL,
help="Skip URL Check inside initvm" )
devel.add_option ("--debug", action="store_true",
diff --git a/elbepack/commands/pkgdiff.py b/elbepack/commands/pkgdiff.py
index 0e1a588..ec14f01 100644
--- a/elbepack/commands/pkgdiff.py
+++ b/elbepack/commands/pkgdiff.py
@@ -24,7 +24,7 @@ import apt_pkg
from optparse import OptionParser
-from elbepack.elbexml import ElbeXML
+from elbepack.elbexml import ElbeXML, ValidationMode
def run_command( argv ):
@@ -42,7 +42,7 @@ def run_command( argv ):
fix_rfs = args[1]
x = os.path.join(gen_rfs, 'etc/elbe_base.xml')
- xml = ElbeXML (x, skip_validate=True, skip_urlcheck=True)
+ xml = ElbeXML (x, skip_validate=True, url_validation=ValidationMode.NO_CHECK)
arch = xml.text ('project/arch', key='arch')
apt_pkg.init_config()
diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py
index cc608f4..daba6f2 100644
--- a/elbepack/daemons/soap/esoap.py
+++ b/elbepack/daemons/soap/esoap.py
@@ -29,6 +29,7 @@ from elbepack.version import elbe_version
from faults import soap_faults
from elbepack.db import Project, User
+from elbepack.elbexml import ValidationMode
from datatypes import SoapProject, SoapFile
from authentication import authenticated_admin, authenticated_uid
@@ -100,7 +101,7 @@ class ESoap (ServiceBase):
if (fname == "source.xml"):
# ensure that the project cache is reloaded
self.app.pm.close_current_project (uid)
- self.app.pm.open_project (uid, builddir, skip_urlcheck=True)
+ self.app.pm.open_project (uid, builddir, url_validation=ValidationMode.NO_CHECK)
self.app.pm.set_current_project_xml (uid, fn)
return -2
@@ -164,7 +165,7 @@ class ESoap (ServiceBase):
@authenticated_uid
@soap_faults
def start_cdrom (self, uid, builddir):
- self.app.pm.open_project (uid, builddir, skip_urlcheck=True)
+ self.app.pm.open_project (uid, builddir, url_validation=ValidationMode.NO_CHECK)
cdrom_fname = os.path.join (builddir, "uploaded_cdrom.iso")
@@ -176,7 +177,7 @@ class ESoap (ServiceBase):
@authenticated_uid
@soap_faults
def append_cdrom (self, uid, builddir, data):
- self.app.pm.open_project (uid, builddir, skip_urlcheck=True)
+ self.app.pm.open_project (uid, builddir, url_validation=ValidationMode.NO_CHECK)
cdrom_fname = os.path.join (builddir, "uploaded_cdrom.iso")
@@ -189,7 +190,7 @@ class ESoap (ServiceBase):
@authenticated_uid
@soap_faults
def finish_cdrom (self, uid, builddir):
- self.app.pm.open_project (uid, builddir, skip_urlcheck=True)
+ self.app.pm.open_project (uid, builddir, url_validation=ValidationMode.NO_CHECK)
self.app.pm.set_current_project_upload_cdrom (uid)
@rpc (String)
@@ -276,18 +277,18 @@ class ESoap (ServiceBase):
@rpc(String, String, _returns=String)
@authenticated_uid
@soap_faults
- def create_project (self, uid, xml, skip_urlcheck):
+ def create_project (self, uid, xml, url_validation):
with NamedTemporaryFile() as fp:
fp.write (binascii.a2b_base64 (xml))
fp.flush ()
- prjid = self.app.pm.create_project (uid, fp.name, skip_urlcheck=skip_urlcheck)
+ prjid = self.app.pm.create_project (uid, fp.name, url_validation=url_validation)
return prjid
@rpc(String, _returns=String)
@authenticated_uid
@soap_faults
- def new_project (self, uid, skip_urlcheck):
+ def new_project (self, uid, url_validation):
return self.app.pm.new_project (uid)
@rpc (String, Integer, _returns=String)
diff --git a/elbepack/db.py b/elbepack/db.py
index 288f59e..120ed5c 100644
--- a/elbepack/db.py
+++ b/elbepack/db.py
@@ -41,7 +41,7 @@ from sqlalchemy.orm.exc import NoResultFound
from sqlalchemy.exc import OperationalError
from elbepack.elbeproject import ElbeProject
-from elbepack.elbexml import (ElbeXML, ValidationError)
+from elbepack.elbexml import (ElbeXML, ValidationError, ValidationMode)
from elbepack.dosunix import dos2unix
Base = declarative_base ()
@@ -278,7 +278,7 @@ class ElbeDB(object):
"cannot set XML file while project %s is busy" %
builddir )
- xml = ElbeXML (xml_file, skip_urlcheck=True) #ValidationError
+ xml = ElbeXML (xml_file, url_validation=ValidationMode.NO_CHECK) #ValidationError
p.name = xml.text ("project/name")
p.version = xml.text ("project/version")
@@ -419,7 +419,7 @@ class ElbeDB(object):
project.xml = xml_str
- def load_project (self, builddir, logpath = None, skip_urlcheck=False):
+ def load_project (self, builddir, logpath = None, url_validation=ValidationMode.CHECK_ALL):
# pass exceptions if hook-scripts can't be loaded (they're optional)
postbuild_file = None
@@ -460,7 +460,7 @@ class ElbeDB(object):
presh_file=presh_file,
postsh_file=postsh_file,
savesh_file=savesh_file,
- skip_urlcheck=skip_urlcheck)
+ url_validation=url_validation)
except NoResultFound:
raise ElbeDBError(
"project %s is not registered in the database" %
@@ -574,7 +574,7 @@ class ElbeDB(object):
" set_project_version: invalid status: " + p.status )
xmlpath = os.path.join( builddir, "source.xml" )
- xml = ElbeXML( xmlpath, skip_urlcheck=True )
+ xml = ElbeXML( xmlpath, url_validation=ValidationMode.NO_CHECK )
if not new_version is None:
xml.node( "/project/version" ).set_text( new_version )
@@ -608,7 +608,7 @@ class ElbeDB(object):
assert p.status == "busy"
sourcexmlpath = os.path.join( builddir, "source.xml" )
- sourcexml = ElbeXML( sourcexmlpath, skip_urlcheck=True )
+ sourcexml = ElbeXML( sourcexmlpath, url_validation=ValidationMode.NO_CHECK )
version = sourcexml.text( "project/version" )
if s.query( ProjectVersion ).\
diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py
index 73e2fad..7197b5f 100644
--- a/elbepack/elbeproject.py
+++ b/elbepack/elbeproject.py
@@ -24,7 +24,7 @@ import io
from elbepack.asciidoclog import ASCIIDocLog, StdoutLog
from elbepack.shellhelper import CommandError
-from elbepack.elbexml import ElbeXML, NoInitvmNode, ValidationError
+from elbepack.elbexml import ElbeXML, NoInitvmNode, ValidationError, ValidationMode
from elbepack.rfs import BuildEnv
from elbepack.rpcaptcache import get_rpcaptcache
from elbepack.efilesystem import TargetFs
@@ -57,7 +57,7 @@ class AptCacheCommitError(Exception):
class ElbeProject (object):
def __init__ (self, builddir, xmlpath = None, logpath = None, name = None,
override_buildtype = None, skip_validate = False,
- skip_urlcheck = False, rpcaptcache_notifier = None,
+ url_validation = ValidationMode.CHECK_ALL, rpcaptcache_notifier = None,
private_data = None, postbuild_file = None, presh_file = None,
postsh_file = None, savesh_file = None):
self.builddir = os.path.abspath(str(builddir))
@@ -67,7 +67,7 @@ class ElbeProject (object):
self.name = name
self.override_buildtype = override_buildtype
self.skip_validate = skip_validate
- self.skip_urlcheck = skip_urlcheck
+ self.url_validation = url_validation
self.postbuild_file = postbuild_file
self.presh_file = presh_file
self.postsh_file = postsh_file
@@ -90,11 +90,11 @@ class ElbeProject (object):
# file of the project
if xmlpath:
self.xml = ElbeXML( xmlpath, buildtype=override_buildtype,
- skip_validate=skip_validate, skip_urlcheck=skip_urlcheck )
+ skip_validate=skip_validate, url_validation=url_validation )
else:
sourcexmlpath = os.path.join( self.builddir, "source.xml" )
self.xml = ElbeXML( sourcexmlpath, buildtype=override_buildtype,
- skip_validate=skip_validate, skip_urlcheck=skip_urlcheck )
+ skip_validate=skip_validate, url_validation=url_validation )
self.arch = self.xml.text( "project/arch", key="arch" )
self.codename = self.xml.text( "project/suite" )
@@ -206,6 +206,15 @@ class ElbeProject (object):
# Write the log header
self.write_log_header()
+ # Validate Apt Sources
+ m = ValidationMode.NO_CHECK
+ if build_bin:
+ m = ValidationMode.CHECK_BINARIES
+ if build_sources:
+ m = ValidationMode.CHECK_ALL
+ self.xml.validate_apt_sources ( m , self.arch )
+
+
if (self.xml.has('target/pbuilder') and not skip_pbuild):
if not os.path.exists ( os.path.join (self.builddir, "pbuilder") ):
self.create_pbuilder ()
@@ -458,7 +467,7 @@ class ElbeProject (object):
newxml = ElbeXML( xmlpath, buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
- skip_urlcheck=self.skip_urlcheck )
+ url_validation=self.url_validation )
# New XML file has to have the same architecture
oldarch = self.xml.text( "project/arch", key="arch" )
@@ -511,7 +520,7 @@ class ElbeProject (object):
source = self.xml
try:
initxml = ElbeXML( "/var/cache/elbe/source.xml",
- skip_validate=self.skip_validate, skip_urlcheck=True )
+ skip_validate=self.skip_validate, url_validation=ValidationMode.NO_CHECK )
self.xml.get_initvmnode_from( initxml )
except ValidationError as e:
self.log.printo( "/var/cache/elbe/source.xml validation failed" )
@@ -528,7 +537,7 @@ class ElbeProject (object):
source = ElbeXML( sourcepath,
buildtype=self.override_buildtype,
skip_validate=self.skip_validate,
- skip_urlcheck=self.skip_urlcheck )
+ url_validation=self.url_validation )
self.xml.get_debootstrappkgs_from( source )
try:
diff --git a/elbepack/elbexml.py b/elbepack/elbexml.py
index 5e6f889..36283a4 100644
--- a/elbepack/elbexml.py
+++ b/elbepack/elbexml.py
@@ -49,8 +49,13 @@ class ValidationError(Exception):
class NoInitvmNode(Exception):
pass
+class ValidationMode():
+ NO_CHECK = True
+ CHECK_BINARIES = 2
+ CHECK_ALL = False
+
class ElbeXML(object):
- def __init__(self, fname, buildtype=None, skip_validate=False, skip_urlcheck=False):
+ def __init__(self, fname, buildtype=None, skip_validate=False, url_validation=ValidationMode.NO_CHECK):
if not skip_validate:
validation = validate_xml (fname)
if len (validation) != 0:
@@ -68,8 +73,8 @@ class ElbeXML(object):
buildtype = "nodefaults"
self.defs = ElbeDefaults(buildtype)
- if not skip_validate and not skip_urlcheck:
- self.validate_apt_sources ()
+ if not skip_validate:
+ self.validate_apt_sources (url_validation, buildtype)
def text(self, txt, key=None):
@@ -142,7 +147,7 @@ class ElbeXML(object):
return mirror.replace("LOCALMACHINE", "10.0.2.2")
- def validate_apt_sources (self):
+ def validate_apt_sources (self, url_validation, buildtype):
slist = self.create_apt_sources_list ()
sources_lines = slist.split ('\n')
@@ -158,9 +163,16 @@ class ElbeXML(object):
elif l.startswith ("deb ") or l.startswith ("deb-src "):
lsplit = l.split (" ")
if lsplit[2].endswith('/'):
- urls.append ("%s/%sRelease" % (lsplit[1], lsplit[2]))
+ s = "%s/%s" % (lsplit[1], lsplit[2])
else:
- urls.append ("%s/dists/%s/Release" % (lsplit[1], lsplit[2]))
+ s = "%s/dists/%s/" % (lsplit[1], lsplit[2])
+
+ urls.append(s + "Release")
+ if url_validation == ValidationMode.CHECK_ALL:
+ urls.append(s + lsplit[3] + "/source/Release")
+ urls.append(s + lsplit[3] + "/binary-%s/Release" % buildtype)
+ elif url_validation == ValidationMode.CHECK_BINARIES:
+ urls.append(s + lsplit[3] + "/binary-%s/Release" % buildtype)
if not self.prj:
return
diff --git a/elbepack/initvmaction.py b/elbepack/initvmaction.py
index bbbd76c..ed062b5 100644
--- a/elbepack/initvmaction.py
+++ b/elbepack/initvmaction.py
@@ -25,7 +25,7 @@ from elbepack.treeutils import etree
from elbepack.directories import elbe_exe
from elbepack.shellhelper import CommandError, system, command_out_stderr
from elbepack.filesystem import wdfs, TmpdirFilesystem, Filesystem
-from elbepack.elbexml import ElbeXML, ValidationError
+from elbepack.elbexml import ElbeXML, ValidationError, ValidationMode
from tempfile import NamedTemporaryFile
@@ -273,7 +273,7 @@ class CreateAction(InitVMAction):
sys.exit (20)
try:
- exml = ElbeXML (tmp.fname ('source.xml'), skip_urlcheck=True)
+ exml = ElbeXML (tmp.fname ('source.xml'), url_validation=ValidationMode.NO_CHECK)
except ValidationError as e:
print ('Iso image does contain a source.xml file.', file=sys.stderr)
print ('But that xml does not validate correctly', file=sys.stderr)
@@ -477,7 +477,7 @@ class SubmitAction(InitVMAction):
if args[0].endswith ('.xml'):
# We have an xml file, use that for elbe init
xmlfile = args[0]
- skip_urlcheck = ''
+ url_validation = ''
elif args[0].endswith ('.iso'):
# We have an iso image, extract xml from there.
tmp = TmpdirFilesystem ()
@@ -493,7 +493,7 @@ class SubmitAction(InitVMAction):
sys.exit (20)
try:
- exml = ElbeXML (tmp.fname ('source.xml'), skip_urlcheck=True)
+ exml = ElbeXML (tmp.fname ('source.xml'), url_validation=ValidationMode.NO_CHECK)
except ValidationError as e:
print ('Iso image does contain a source.xml file.', file=sys.stderr)
print ('But that xml does not validate correctly', file=sys.stderr)
@@ -505,7 +505,7 @@ class SubmitAction(InitVMAction):
print ('Image was generated using Elbe Version %s' % exml.get_elbe_version ())
xmlfile = tmp.fname ('source.xml')
- skip_urlcheck = '--skip-urlcheck'
+ url_validation = '--skip-urlcheck'
cdrom = args[0]
else:
print ('Unknown file ending (use either xml or iso)', file=sys.stderr)
diff --git a/elbepack/projectmanager.py b/elbepack/projectmanager.py
index 6b17fc9..05cc040 100644
--- a/elbepack/projectmanager.py
+++ b/elbepack/projectmanager.py
@@ -34,6 +34,7 @@ from elbepack.asyncworker import SaveVersionJob, CheckoutVersionJob
from elbepack.asyncworker import APTUpdUpgrJob, BuildSysrootJob
from elbepack.asyncworker import PdebuildJob, CreatePbuilderJob
from elbepack.asyncworker import BuildChrootTarJob
+from elbepack.elbexml import ValidationMode
class ProjectManagerError(Exception):
def __init__ (self, message):
@@ -83,7 +84,7 @@ class ProjectManager(object):
self.db.create_project( builddir, owner_id=userid )
return builddir
- def create_project (self, userid, xml_file, skip_urlcheck=False):
+ def create_project (self, userid, xml_file, url_validation=ValidationMode.CHECK_ALL):
subdir = str(uuid4())
builddir = path.join( self.basepath, subdir )
@@ -102,14 +103,14 @@ class ProjectManager(object):
# Open the new project
logpath = path.join( builddir, "log.txt" )
- ep = self.db.load_project( builddir, logpath, skip_urlcheck=skip_urlcheck )
+ ep = self.db.load_project( builddir, logpath, url_validation=url_validation )
self.userid2project[ userid ] = ep
self.builddir2userid[ builddir ] = userid
return builddir
- def open_project (self, userid, builddir, skip_urlcheck=False):
+ def open_project (self, userid, builddir, url_validation=ValidationMode.CHECK_ALL):
self._check_project_permission( userid, builddir )
with self.lock:
@@ -129,7 +130,7 @@ class ProjectManager(object):
# Load project from the database
logpath = path.join( builddir, "log.txt" )
- ep = self.db.load_project( builddir, logpath, skip_urlcheck=skip_urlcheck )
+ ep = self.db.load_project( builddir, logpath, url_validation=url_validation )
# Add project to our dictionaries
self.userid2project[ userid ] = ep
diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
index bc38a61..1a98f91 100644
--- a/elbepack/soapclient.py
+++ b/elbepack/soapclient.py
@@ -30,7 +30,7 @@ import sys
import os
from elbepack.filesystem import Filesystem
-from elbepack.elbexml import ElbeXML
+from elbepack.elbexml import ElbeXML, ValidationMode
def set_suds_debug(debug):
import logging
@@ -221,7 +221,7 @@ class SetXmlAction(ClientAction):
filename = args[1]
try:
- x = ElbeXML (filename, skip_validate=True, skip_urlcheck=True)
+ x = ElbeXML (filename, skip_validate=True, url_validation=ValidationMode.NO_CHECK)
except IOError as e:
print ("%s is not a valid elbe xml file" % filename)
sys.exit (20)
--
2.1.4
More information about the elbe-devel
mailing list