[elbe-devel] [PATCH v2 00/24] SDK generator
Manuel Traut
manuel.traut at linutronix.de
Fri Feb 9 21:52:17 CET 2018
Hi,
the SDK generator patchset is merged into devel/elbe-3.0 and is available since
release/v2.9.8 that was just uploaded to debian.linutronix.de/elbe-testing
Regards,
Manuel
On Thu, Feb 08, 2018 at 02:16:46PM +0100, Manuel Traut wrote:
> Hi,
>
> this is v2 of the SDK geneartor patch set. This time i merged the preparation
> patches into the same queue. The queue is based on devel/elbe-3.0.
>
> An older version of the patches got a private review from torbenh. I included
> the reviewed-by in this queue. I also tried to address some points from the
> review described more detailed in 'changes since v1'.
>
> changes since v1 (full diff against v1 is on the bottom of this mail):
>
> * include names and mail addresses of all copyright holders of
> toolchain-shar-extract.sh.mako in debian/copyright (instead of just company
> names)
> * fix GPL-2 vs. 3 typo in debian/copyright
> * use write_pack_template instead of reimplementing it
> * make a difference between triplet and sdkarch. sdkarch is used to construct
> the name of the cross-gcc package installed to the host sysroot
> * remove returns after raise in download_pkg()
> * implement optional log to file in download_pkg()
> * if debootstrap is called on existing rfs, do not only clean the target dir,
> but also create the initial set of files
> * use a set for ignorepkgs in mark_install_devpkgs instead of complicated
> exception handling needed for lists
>
> Manuel Traut (24):
> soap/wait_busy: don't drop exception silently
> db: use same codingstyle for all files to add
> elbeproject: build_sysroot rework
> pkgutils: split get_initrd_uri function
> pkgutils: rework copy_kinitrd function
> pkgutils: raise an exception if pkg not found
> pkgutils: remove useless returns
> rfs: debootstrap always into clean target dir
> rpcaptcache: ValueError is not derived from BaseException
> run virtapt in a forked process
> build sdk: create infrastructure
> toolchain-extract.sh: import file from yocto
> toolchain-sh-extract.sh: improve copyright
> debian/copyright: add toolchain-extract.sh
> toolchain-extract.sh: add some sections
> toolchain-extract.sh: convert to work with mako
> elbeproject: implement build_sdk function
> db: add sdk installer as output file
> sdk/environment-setup: add file
> elbeproject/sdk: create env-setup file
> elbeproject: add a test to generate the SDK scripts
> sdk/environment-setup: dont hardcode sdk path
> elbeproject/sdk: encode project details into sdk
> elbeproject: dont reimplement write_pack_template
>
> debian/copyright | 20 ++
> elbepack/asyncworker.py | 25 ++
> elbepack/commands/buildsdk.py | 39 +++
> elbepack/daemons/soap/esoap.py | 7 +
> elbepack/db.py | 91 +++---
> elbepack/elbeproject.py | 145 +++++++--
> elbepack/makofiles/environment-setup-elbe.mako | 93 ++++++
> elbepack/makofiles/toolchain-shar-extract.sh.mako | 342 ++++++++++++++++++++++
> elbepack/pkgutils.py | 170 +++++------
> elbepack/projectmanager.py | 21 +-
> elbepack/rfs.py | 8 +
> elbepack/rpcaptcache.py | 24 +-
> elbepack/soapclient.py | 30 +-
> elbepack/virtapt.py | 54 ++++
> elbepack/xmldefaults.py | 3 +-
> 15 files changed, 884 insertions(+), 188 deletions(-)
> create mode 100644 elbepack/commands/buildsdk.py
> create mode 100644 elbepack/makofiles/environment-setup-elbe.mako
> create mode 100644 elbepack/makofiles/toolchain-shar-extract.sh.mako
>
> full diff since v1:
>
> diff --git a/debian/copyright b/debian/copyright
> index 53c2e7ec..38088a6e 100644
> --- a/debian/copyright
> +++ b/debian/copyright
> @@ -3,7 +3,18 @@ Upstream-Name: elbe
> Source: https://github.com/Linutronix/elbe/
>
> Files: elbepack/makofiles/toolchain-shar-extract.sh.mako
> -Copyright: Windriver, Intel Corporation, Linux Foundation
> +Copyright: Aníbal Limón <anibal.limon at linux.intel.com>
> + Brendan Le Foll <brendan.le.foll at intel.com>
> + Ed Bartosh <ed.bartosh at linux.intel.com>
> + George Nita <george.nita at enea.com>
> + Maxin B. John <maxin.john at intel.com>
> + Paul Eggleton <paul.eggleton at linux.intel.com>
> + Qi.Chen at windriver.com <Qi.Chen at windriver.com>
> + Randy Witt <randy.e.witt at linux.intel.com>
> + Richard Purdie <richard.purdie at linuxfoundation.org>
> + Robert Yang <liezhi.yang at windriver.com>
> + Todor Minchev <todor.minchev at linux.intel.com>
> + Wenlin Kang <wenlin.kang at windriver.com>
> License: GPL-2
>
> Files: *
> @@ -12,7 +23,7 @@ License: GPL-3
>
> License: GPL-2
> On Debian systems, the full text of the GNU General Public
> - License version 3 can be found in the file
> + License version 2 can be found in the file
> `/usr/share/common-licenses/GPL-2'.
>
> License: GPL-3
> diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py
> index a044b951..1434c5e6 100644
> --- a/elbepack/elbeproject.py
> +++ b/elbepack/elbeproject.py
> @@ -44,7 +44,7 @@ from elbepack.repomanager import ProjectRepo
> from elbepack.config import cfg
> from elbepack.pkgutils import extract_pkg
> from elbepack.directories import (mako_template_dir, pack_dir)
> -from elbepack.templates import template
> +from elbepack.templates import template, write_pack_template
>
>
> class IncompatibleArchitectureException(Exception):
> @@ -65,6 +65,11 @@ class AptCacheCommitError(Exception):
> Exception.__init__(self, "Error Committing rpcaptcache %s" % msg)
>
>
> +class UnsupportedSDKException(Exception):
> + def __init__(self, triplet):
> + Exception.__init__(self, "SDK for %s currently unsupported" % triplet)
> +
> +
> def test_gen_sdk_scripts():
> os.system("mkdir -p /tmp/test/sdk")
> gen_sdk_scripts('armhf-linux-gnueabihf',
> @@ -90,20 +95,20 @@ def gen_sdk_scripts(triplet, prj_name, prj_version, builddir, sdkpath):
> 'sdk_title': 'ELBE %s' % prj_name,
> 'sdk_version': prj_version,
> }
> - t = os.path.join(mako_template_dir, 'toolchain-shar-extract.sh.mako')
> sdkname = 'setup-elbe-sdk-%s-%s-%s.sh' % (triplet, prj_name, prj_version)
> - with open(os.path.join(builddir, sdkname), 'w') as f:
> - f.write(template(t, sdkvalues))
> -
> - t = os.path.join(mako_template_dir, 'environment-setup-elbe.mako')
> + write_pack_template(os.path.join(builddir, sdkname),
> + 'toolchain-shar-extract.sh.mako',
> + sdkvalues)
> envname = 'environment-setup-elbe-%s-%s-%s' % (triplet,
> prj_name,
> prj_version)
> - with open(os.path.join(sdkpath, envname), 'w') as f:
> - f.write(template(t, sdkvalues))
> + write_pack_template(os.path.join(sdkpath, envname),
> + 'environment-setup-elbe.mako',
> + sdkvalues)
>
> return sdkname
>
> +
> class ElbeProject (object):
> def __init__(
> self,
> @@ -243,8 +248,8 @@ class ElbeProject (object):
> raise AptCacheUpdateError(e)
>
> try:
> - self.get_rpcaptcache().mark_install_devpkgs(ignore_pkgs,
> - ignore_dev_pkgs)
> + self.get_rpcaptcache().mark_install_devpkgs(set(ignore_pkgs),
> + set(ignore_dev_pkgs))
> except SystemError as e:
> self.log.printo("mark install devpkgs failed: %s" % str(e))
> try:
> @@ -279,6 +284,11 @@ class ElbeProject (object):
> def build_sdk(self):
> triplet = self.xml.defs["triplet"]
>
> + try:
> + crosstoolchainpkg = "gcc-%s" % self.xml.defs["sdkarch"]
> + except KeyError:
> + raise UnsupportedSDKException(triplet)
> +
> # build target sysroot including libs and headers for the target
> self.build_sysroot()
> sdktargetpath = os.path.join(self.sdkpath, "sysroots", "target")
> @@ -286,7 +296,6 @@ class ElbeProject (object):
> self.log.do("tar xJf %s/sysroot.tar.xz -C %s" % (self.builddir,
> sdktargetpath))
> # build host sysroot including cross compiler
> - crosstoolchainpkg = "gcc-%s" % self.xml.defs["triplet"]
> hostsysrootpath = os.path.join(self.sdkpath, 'sysroots', 'host')
> self.log.do('mkdir -p "%s"' % hostsysrootpath)
> extract_pkg(self.xml.prj,
> diff --git a/elbepack/pkgutils.py b/elbepack/pkgutils.py
> index 76dcc69d..e062396e 100644
> --- a/elbepack/pkgutils.py
> +++ b/elbepack/pkgutils.py
> @@ -178,16 +178,20 @@ def get_dsc_size(fname):
> return sz
>
>
> -def download_pkg(prj, target_dir, defs, package, arch="default", incl_deps=False):
> +def download_pkg(prj,
> + target_dir,
> + defs,
> + package,
> + arch="default",
> + incl_deps=False,
> + log=None):
>
> try:
> urilist = get_uri(prj, defs, arch, package, incl_deps)
> except KeyError:
> raise NoKinitrdException('no package %s available' % package)
> - return
> except SystemError:
> raise NoKinitrdException('a configured mirror is not reachable')
> - return
> except CommandError as e:
> raise NoKinitrdException("couldn't download package %s" % package)
>
> @@ -216,24 +220,24 @@ def download_pkg(prj, target_dir, defs, package, arch="default", incl_deps=False
> while len(buf) > 0:
> m.update(buf)
> buf = f.read(65536)
> -
> if m.hexdigest() != sha256:
> raise NoKinitrdException('%s failed to verify !!!' % package)
> else:
> - # TODO this should be logged to validation.txt or stderr depending
> - # on use-case
> - print("-----------------------------------------------------")
> - print("WARNING:")
> - print("Using untrusted %s package" % package)
> - print("-----------------------------------------------------")
> + if log:
> + log.printo("WARNING: Using untrusted %s package" % package)
> + else:
> + print("-----------------------------------------------------")
> + print("WARNING:")
> + print("Using untrusted %s package" % package)
> + print("-----------------------------------------------------")
>
> return [y[0] for y in urilist]
>
>
> def extract_pkg(prj, target_dir, defs, package, arch="default",
> - incl_deps=False):
> + incl_deps=False, log=None):
>
> - pkgs = download_pkg(prj, target_dir, defs, package, arch, incl_deps)
> + pkgs = download_pkg(prj, target_dir, defs, package, arch, incl_deps, log)
>
> for package in pkgs:
> ppath = os.path.join(target_dir, "%s.deb" % package)
> diff --git a/elbepack/rfs.py b/elbepack/rfs.py
> index 4916e063..6c2ffa4a 100644
> --- a/elbepack/rfs.py
> +++ b/elbepack/rfs.py
> @@ -43,6 +43,9 @@ class BuildEnv ():
> if clean:
> self.rfs.rmtree("")
>
> + self.gen_base_files()
> +
> + def gen_base_files(self):
> # TODO think about reinitialization if elbe_version differs
> if not self.rfs.isfile("etc/elbe_version"):
> # avoid starting daemons inside the buildenv
> @@ -102,6 +105,7 @@ class BuildEnv ():
> # deboostraping causes errors if running on an existing filesystem
> # ensure to run on an empty directory
> self.rfs.rmtree("/")
> + self.gen_base_files()
>
> primary_mirror = self.xml.get_primary_mirror(
> self.rfs.fname('/cdrom/targetrepo'))
> diff --git a/elbepack/rpcaptcache.py b/elbepack/rpcaptcache.py
> index 975eec0f..7a90e838 100644
> --- a/elbepack/rpcaptcache.py
> +++ b/elbepack/rpcaptcache.py
> @@ -112,16 +112,9 @@ class RPCAPTCache(InChRootObject):
> from_user=from_user)
>
> def mark_install_devpkgs(self, ignore_pkgs, ignore_dev_pkgs):
> - ignore_pkgs.remove('libc6') # we don't want to ignore libc
> - # we don't want to ignore libstdc++
> - try:
> - ignore_pkgs.remove('libstdc++5')
> - except ValueError:
> - pass
> - try:
> - ignore_pkgs.remove('libstdc++6')
> - except ValueError:
> - pass
> + ignore_pkgs.discard('libc6') # we don't want to ignore libc
> + ignore_pkgs.discard('libstdc++5')
> + ignore_pkgs.discard('libstdc++6')
> # list all debian src packages of all installed packages that don't
> # come from debootstrap
> src_list = [
> diff --git a/elbepack/xmldefaults.py b/elbepack/xmldefaults.py
> index 2d357c54..4ac7de92 100644
> --- a/elbepack/xmldefaults.py
> +++ b/elbepack/xmldefaults.py
> @@ -54,7 +54,8 @@ armhf_defaults = {
> "console": "ttyAMA0,115200n1",
> "machine": "versatilepb -cpu cortex-a9",
> "nicmodel": "smc91c111",
> - "triplet": "arm-linux-gnueabihf"
> + "triplet": "arm-linux-gnueabihf",
> + "sdkarch": "arm-linux-gnueabihf"
> }
>
> armhf_linaro48_defaults = {
> @@ -138,7 +139,7 @@ i386_defaults = {
> "console": "ttyS0,115200n1",
> "machine": "pc",
> "nicmodel": "virtio",
> - "triplet": "i686-linux-gnu"
> + "triplet": "i386-linux-gnu"
> }
>
> archindep_defaults = {
>
> --
> 2.15.1
>
More information about the elbe-devel
mailing list