[elbe-devel] [PATCH v2 4/6] debinstaller: implement download of vmlinuz and initrd.gz
Torben Hohn
torben.hohn at linutronix.de
Wed Sep 26 12:13:50 CEST 2018
the debinstaller module shall replace elbe-bootstrap.
it downloads debian installer linux kernel and initrd.gz from
a debian mirror. It does that in a secure manner, iE validating
Release.gpg and SHA256SUMS on the way.
filenames on the mirror:
/debian/dists/jessie/main/installer-amd64/current/images/netboot/debian-installer/amd64/linux
/debian/dists/jessie/main/installer-amd64/current/images/netboot/debian-installer/amd64/initrd.gz
/debian/dists/jessie/main/installer-amd64/current/images/cdrom/initrd.gz
Although this functionality is also provided by apt,
we implement it here in pure python, because outside
of the initvm, we can not rely on apt being available.
The initrd and vmlinuz are stored in the initvm in /var/cache/elbe/installer.
Also put them on bin-cdrom.iso, and reuse them, when an elbe build
is run from cdrom.
With this code in Place, we can remove the fallback
Code in pkgutils. We also will never see initvm build
failures due to new debian Releases requiring new
installer/elbe-bootstrap versions.
Signed-off-by: Torben Hohn <torben.hohn at linutronix.de>
---
debian/python-elbe-common.install | 1 +
debian/python3-elbe-common.install | 1 +
elbepack/cdroms.py | 7 ++
elbepack/commands/init.py | 4 +-
elbepack/debinstaller.py | 225 +++++++++++++++++++++++++++++++++++++
elbepack/init/Makefile.mako | 2 +
elbepack/init/init-elbe.sh.mako | 4 +
elbepack/pkgutils.py | 26 -----
8 files changed, 242 insertions(+), 28 deletions(-)
create mode 100644 elbepack/debinstaller.py
diff --git a/debian/python-elbe-common.install b/debian/python-elbe-common.install
index bf51499c..e2788e38 100644
--- a/debian/python-elbe-common.install
+++ b/debian/python-elbe-common.install
@@ -9,6 +9,7 @@
./usr/lib/python2.*/*-packages/elbepack/aptprogress.py
./usr/lib/python2.*/*-packages/elbepack/archivedir.py
./usr/lib/python2.*/*-packages/elbepack/config.py
+./usr/lib/python2.*/*-packages/elbepack/debinstaller.py
./usr/lib/python2.*/*-packages/elbepack/default-preseed.xml
./usr/lib/python2.*/*-packages/elbepack/directories.py
./usr/lib/python2.*/*-packages/elbepack/dosunix.py
diff --git a/debian/python3-elbe-common.install b/debian/python3-elbe-common.install
index c2a0aec7..e379ceb9 100644
--- a/debian/python3-elbe-common.install
+++ b/debian/python3-elbe-common.install
@@ -9,6 +9,7 @@
./usr/lib/python3.*/*-packages/elbepack/aptprogress.py
./usr/lib/python3.*/*-packages/elbepack/archivedir.py
./usr/lib/python3.*/*-packages/elbepack/config.py
+./usr/lib/python3.*/*-packages/elbepack/debinstaller.py
./usr/lib/python3.*/*-packages/elbepack/default-preseed.xml
./usr/lib/python3.*/*-packages/elbepack/directories.py
./usr/lib/python3.*/*-packages/elbepack/dosunix.py
diff --git a/elbepack/cdroms.py b/elbepack/cdroms.py
index 9e267228..19790e6b 100644
--- a/elbepack/cdroms.py
+++ b/elbepack/cdroms.py
@@ -7,6 +7,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import os
+from shutil import copyfile
from apt.package import FetchError
from apt import Cache
@@ -258,6 +259,12 @@ def mk_binary_cdrom(
# write source xml onto cdrom
xml.xml.write(repo_fs.fname('source.xml'))
+ # copy initvm-cdrom.gz and vmlinuz
+ copyfile('/var/cache/elbe/installer/initrd-cdrom.gz',
+ repo_fs.fname('initrd-cdrom.gz'))
+ copyfile('/var/cache/elbe/installer/vmlinuz',
+ repo_fs.fname('vmlinuz'))
+
target_repo_fs = Filesystem(target_repo_path)
target_repo_fs.write_file(".aptignr", 0o644, "")
diff --git a/elbepack/commands/init.py b/elbepack/commands/init.py
index 9445c69d..61a30447 100644
--- a/elbepack/commands/init.py
+++ b/elbepack/commands/init.py
@@ -15,7 +15,7 @@ from optparse import OptionParser
from elbepack.treeutils import etree
from elbepack.validate import validate_xml
-from elbepack.pkgutils import copy_kinitrd, NoKinitrdException
+from elbepack.debinstaller import copy_kinitrd, NoKinitrdException
from elbepack.xmldefaults import ElbeDefaults
from elbepack.version import elbe_version
from elbepack.templates import write_template, get_initvm_preseed
@@ -165,7 +165,7 @@ def run_command(argv):
os.putenv("no_proxy", "localhost,127.0.0.1")
try:
- copy_kinitrd(xml.node("/initvm"), out_path, defs, arch="amd64")
+ copy_kinitrd(xml.node("/initvm"), out_path)
except NoKinitrdException as e:
print("Failure to download kernel/initrd debian Package:")
print("")
diff --git a/elbepack/debinstaller.py b/elbepack/debinstaller.py
new file mode 100644
index 00000000..c02a4978
--- /dev/null
+++ b/elbepack/debinstaller.py
@@ -0,0 +1,225 @@
+# ELBE - Debian Based Embedded Rootfilesystem Builder
+# Copyright (c) 2018 Torben Hohn <torben.hohn at linutronix.de>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import print_function
+
+import sys
+import os
+import re
+
+from urllib2 import urlopen
+
+from shutil import copyfileobj, copyfile
+from gpgme import Context
+
+from elbepack.filesystem import TmpdirFilesystem
+from elbepack.gpg import OverallStatus, check_signature
+from elbepack.shellhelper import CommandError, system
+from elbepack.hashes import HashValidator, HashValidationFailed
+
+
+class InvalidSignature(Exception):
+ pass
+
+class NoKinitrdException(Exception):
+ pass
+
+class ReleaseFile(HashValidator):
+ def __init__(self, base_url, fname, fname_list):
+
+ HashValidator.__init__(self, base_url)
+
+ header_re = re.compile(r'(\w+):(.*)')
+ hash_re = re.compile(r' ([0-9a-f]+)\s+([0-9]+)\s+(\S+)')
+ current_header = ''
+
+ with open(fname, 'r') as fp:
+ for l in fp.readlines():
+ m = header_re.match(l)
+ if m:
+ # line contains an rfc822 Header,
+ # remember it.
+ current_header = m.group(1)
+ continue
+
+ m = hash_re.match(l)
+ if m:
+ # line contains a hash entry.
+ # check filename, whether we are interested in it
+ if m.group(3) in fname_list:
+ self.insert_fname_hash(current_header, m.group(3), m.group(1))
+
+
+class SHA256SUMSFile(HashValidator):
+ def __init__(self, base_url, fname, fname_list):
+
+ HashValidator.__init__(self, base_url)
+
+ hash_re = re.compile(r'([0-9a-f]+)\s+(\S+)')
+
+ with open(fname, 'r') as fp:
+ for l in fp.readlines():
+ m = hash_re.match(l)
+ if m:
+ # line contains a hash entry.
+ # check filename, whether we are interested in it
+ if m.group(2) in fname_list:
+ self.insert_fname_hash("SHA256", m.group(2), m.group(1))
+
+
+def setup_apt_keyring(gpg_home, keyring_fname):
+ ring_path = os.path.join(gpg_home, keyring_fname)
+ if not os.path.isdir("/etc/apt/trusted.gpg.d"):
+ print("/etc/apt/trusted.gpg.d doesn't exist")
+ print("apt-get install debian-archive-keyring may "
+ "fix this problem")
+ sys.exit(20)
+
+ if os.path.exists("/etc/apt/trusted.gpg"):
+ system('cp /etc/apt/trusted.gpg "%s"' % ring_path)
+
+ gpg_options = '--keyring "%s" --no-auto-check-trustdb ' \
+ '--trust-model always --no-default-keyring ' \
+ '--homedir "%s"' % (ring_path, gpg_home)
+
+ trustkeys = os.listdir("/etc/apt/trusted.gpg.d")
+ for key in trustkeys:
+ print("Import %s: " % key)
+ try:
+ system('gpg %s --import "%s"' % (
+ gpg_options,
+ os.path.join("/etc/apt/trusted.gpg.d", key)))
+ except CommandError:
+ print('adding keyring "%s" to keyring "%s" failed' % (key, ring_path))
+
+def download(url, local_fname):
+ try:
+ rf = urlopen(url, None, 10)
+ with open(local_fname, "w") as wf:
+ copyfileobj(rf, wf)
+ finally:
+ rf.close()
+
+
+def download_release(tmp, base_url):
+
+ # setup gpg context, for verifying
+ # the Release.gpg signature.
+ os.environ['GNUPGHOME'] = tmp.fname('/')
+ ctx = Context()
+
+ # download the Relase file to a tmp file,
+ # because we need it 2 times
+ download(base_url + "Release", tmp.fname('Release'))
+
+ # validate signature.
+ # open downloaded plaintext file, and
+ # use the urlopen object of the Release.gpg
+ # directtly.
+ try:
+ sig = urlopen(base_url + 'Release.gpg', None, 10)
+ with tmp.open("Release", "r") as signed:
+
+ overall_status = OverallStatus()
+
+ # verify detached signature
+ sigs = ctx.verify(sig, signed, None)
+
+ for s in sigs:
+ status = check_signature(ctx, s)
+ overall_status.add(status)
+
+ if overall_status.to_exitcode():
+ raise InvalidSignature('Failed to verify Release file')
+ finally:
+ sig.close()
+
+def download_kinitrd(tmp, suite, mirror):
+ base_url = "%s/dists/%s/" % (
+ mirror.replace("LOCALMACHINE", "localhost"), suite)
+ installer_path = "main/installer-amd64/current/images/"
+
+ setup_apt_keyring(tmp.fname('/'), 'pubring.gpg')
+
+ # download release file and check
+ # signature
+ download_release(tmp, base_url)
+
+ # parse Release file, and remember hashvalues
+ # we are interested in
+ interesting = [installer_path + 'SHA256SUMS']
+ release_file = ReleaseFile(base_url, tmp.fname('Release'), interesting)
+
+ # now download and validate SHA256SUMS
+ release_file.download_and_validate_file(
+ installer_path + 'SHA256SUMS',
+ tmp.fname('SHA256SUMS'))
+
+ # now we have a valid SHA256SUMS file
+ # parse it
+ interesting = ['./cdrom/initrd.gz',
+ './cdrom/vmlinuz',
+ './netboot/debian-installer/amd64/initrd.gz',
+ './netboot/debian-installer/amd64/linux']
+ sha256_sums = SHA256SUMSFile(
+ base_url + installer_path,
+ tmp.fname('SHA256SUMS'),
+ interesting)
+
+ # and then download the files we actually want
+ for p,ln in zip(interesting, ['initrd-cdrom.gz',
+ 'linux-cdrom',
+ 'initrd.gz',
+ 'vmlinuz']):
+ sha256_sums.download_and_validate_file(
+ p,
+ tmp.fname(ln))
+
+
+
+def get_primary_mirror(prj):
+ if prj.has("mirror/primary_host"):
+ m = prj.node("mirror")
+
+ mirror = m.text("primary_proto") + "://"
+ mirror += m.text("primary_host") + "/"
+ mirror += m.text("primary_path")
+ else:
+ raise NoKinitrdException("Broken xml file: no cdrom and no primary host")
+
+ return mirror
+
+def copy_kinitrd(prj, target_dir):
+
+ suite = prj.text("suite")
+
+ try:
+ tmp = TmpdirFilesystem()
+ if prj.has("mirror/cdrom"):
+ system('7z x -o%s "%s" initrd-cdrom.gz vmlinuz' %
+ (tmp.fname('/'), prj.text("mirror/cdrom")))
+
+ # initrd.gz needs to be cdrom version !
+ copyfile(tmp.fname("initrd-cdrom.gz"),
+ os.path.join(target_dir, "initrd.gz"))
+ else:
+ mirror = get_primary_mirror(prj)
+ download_kinitrd(tmp, suite, mirror)
+
+ copyfile(tmp.fname("initrd.gz"),
+ os.path.join(target_dir, "initrd.gz"))
+
+ copyfile(tmp.fname("initrd-cdrom.gz"),
+ os.path.join(target_dir, "initrd-cdrom.gz"))
+
+ copyfile(tmp.fname("vmlinuz"),
+ os.path.join(target_dir, "vmlinuz"))
+
+ except IOError as e:
+ raise NoKinitrdException('IoError %s' % e.message)
+ except InvalidSignature as e:
+ raise NoKinitrdException('InvalidSignature %s' % e.message)
+ except HashValidationFailed as e:
+ raise NoKinitrdException('HashValidationFailed %s' % e.message)
diff --git a/elbepack/init/Makefile.mako b/elbepack/init/Makefile.mako
index 06c0e0e5..4c2f323b 100644
--- a/elbepack/init/Makefile.mako
+++ b/elbepack/init/Makefile.mako
@@ -62,6 +62,8 @@ all: .stamps/stamp-install-initial-image
cp .elbe-in/source.xml tmp-tree/
mkdir -p tmp-tree/usr/share/keyrings
-cp .elbe-in/*.gpg tmp-tree/usr/share/keyrings
+ cp .elbe-in/initrd-cdrom.gz tmp-tree/
+ cp .elbe-in/vmlinuz tmp-tree/
% if opt.devel:
cp .elbe-in/elbe-devel.tar.bz2 tmp-tree/
% endif
diff --git a/elbepack/init/init-elbe.sh.mako b/elbepack/init/init-elbe.sh.mako
index 4234ec74..b29480dc 100644
--- a/elbepack/init/init-elbe.sh.mako
+++ b/elbepack/init/init-elbe.sh.mako
@@ -24,6 +24,10 @@ cp /etc/apt/apt.conf /buildenv/etc/apt/apt.conf.d/50elbe
ln -s /lib/systemd/system/serial-getty at .service /buildenv/etc/systemd/system/getty.target.wants/serial-getty at ttyS0.service
% endif
+mkdir /buildenv/var/cache/elbe/installer
+cp initrd-cdrom.gz /buildenv/var/cache/elbe/installer
+cp vmlinuz /buildenv/var/cache/elbe/installer
+
% if opt.devel:
mkdir /buildenv/var/cache/elbe/devel
tar xj -f elbe-devel.tar.bz2 -C /buildenv/var/cache/elbe/devel
diff --git a/elbepack/pkgutils.py b/elbepack/pkgutils.py
index 0a68a819..08b6003d 100644
--- a/elbepack/pkgutils.py
+++ b/elbepack/pkgutils.py
@@ -48,8 +48,6 @@ except ImportError as e:
virtapt_imported = False
-class NoKinitrdException(Exception):
- pass
def get_sources_list(prj, defs):
@@ -287,27 +285,3 @@ def extract_pkg(prj, target_dir, defs, package, arch="default",
system('ar p "%s" data.tar.xz | tar xJ -C "%s"' % (ppath,
target_dir))
system('rm -f "%s"' % ppath)
-
-
-def copy_kinitrd(prj, target_dir, defs, arch="default"):
-
- target_pkg = get_initrd_pkg(prj, defs)
-
- try:
- tmpdir = mkdtemp()
- extract_pkg(prj, tmpdir, defs, target_pkg, arch)
-
- # copy is done twice, because paths in elbe-bootstarp_1.0 and 0.9
- # differ
- initrd = os.path.join(tmpdir, 'var', 'lib', 'elbe', 'initrd')
- if prj.has("mirror/cdrom"):
- system('cp "%s" "%s"' % (os.path.join(initrd, 'initrd-cdrom.gz'),
- os.path.join(target_dir, "initrd.gz")))
- else:
- system('cp "%s" "%s"' % (os.path.join(initrd, 'initrd.gz'),
- os.path.join(target_dir, "initrd.gz")))
-
- system('cp "%s" "%s"' % (os.path.join(initrd, 'vmlinuz'),
- os.path.join(target_dir, "vmlinuz")))
- finally:
- system('rm -rf "%s"' % tmpdir)
--
2.11.0
More information about the elbe-devel
mailing list