[elbe-devel] [PATCH v2 00/24] SDK generator
Manuel Traut
manut at linutronix.de
Thu Feb 8 14:16:46 CET 2018
Hi,
this is v2 of the SDK geneartor patch set. This time i merged the preparation
patches into the same queue. The queue is based on devel/elbe-3.0.
An older version of the patches got a private review from torbenh. I included
the reviewed-by in this queue. I also tried to address some points from the
review described more detailed in 'changes since v1'.
changes since v1 (full diff against v1 is on the bottom of this mail):
* include names and mail addresses of all copyright holders of
toolchain-shar-extract.sh.mako in debian/copyright (instead of just company
names)
* fix GPL-2 vs. 3 typo in debian/copyright
* use write_pack_template instead of reimplementing it
* make a difference between triplet and sdkarch. sdkarch is used to construct
the name of the cross-gcc package installed to the host sysroot
* remove returns after raise in download_pkg()
* implement optional log to file in download_pkg()
* if debootstrap is called on existing rfs, do not only clean the target dir,
but also create the initial set of files
* use a set for ignorepkgs in mark_install_devpkgs instead of complicated
exception handling needed for lists
Manuel Traut (24):
soap/wait_busy: don't drop exception silently
db: use same codingstyle for all files to add
elbeproject: build_sysroot rework
pkgutils: split get_initrd_uri function
pkgutils: rework copy_kinitrd function
pkgutils: raise an exception if pkg not found
pkgutils: remove useless returns
rfs: debootstrap always into clean target dir
rpcaptcache: ValueError is not derived from BaseException
run virtapt in a forked process
build sdk: create infrastructure
toolchain-extract.sh: import file from yocto
toolchain-sh-extract.sh: improve copyright
debian/copyright: add toolchain-extract.sh
toolchain-extract.sh: add some sections
toolchain-extract.sh: convert to work with mako
elbeproject: implement build_sdk function
db: add sdk installer as output file
sdk/environment-setup: add file
elbeproject/sdk: create env-setup file
elbeproject: add a test to generate the SDK scripts
sdk/environment-setup: dont hardcode sdk path
elbeproject/sdk: encode project details into sdk
elbeproject: dont reimplement write_pack_template
debian/copyright | 20 ++
elbepack/asyncworker.py | 25 ++
elbepack/commands/buildsdk.py | 39 +++
elbepack/daemons/soap/esoap.py | 7 +
elbepack/db.py | 91 +++---
elbepack/elbeproject.py | 145 +++++++--
elbepack/makofiles/environment-setup-elbe.mako | 93 ++++++
elbepack/makofiles/toolchain-shar-extract.sh.mako | 342 ++++++++++++++++++++++
elbepack/pkgutils.py | 170 +++++------
elbepack/projectmanager.py | 21 +-
elbepack/rfs.py | 8 +
elbepack/rpcaptcache.py | 24 +-
elbepack/soapclient.py | 30 +-
elbepack/virtapt.py | 54 ++++
elbepack/xmldefaults.py | 3 +-
15 files changed, 884 insertions(+), 188 deletions(-)
create mode 100644 elbepack/commands/buildsdk.py
create mode 100644 elbepack/makofiles/environment-setup-elbe.mako
create mode 100644 elbepack/makofiles/toolchain-shar-extract.sh.mako
full diff since v1:
diff --git a/debian/copyright b/debian/copyright
index 53c2e7ec..38088a6e 100644
--- a/debian/copyright
+++ b/debian/copyright
@@ -3,7 +3,18 @@ Upstream-Name: elbe
Source: https://github.com/Linutronix/elbe/
Files: elbepack/makofiles/toolchain-shar-extract.sh.mako
-Copyright: Windriver, Intel Corporation, Linux Foundation
+Copyright: Aníbal Limón <anibal.limon at linux.intel.com>
+ Brendan Le Foll <brendan.le.foll at intel.com>
+ Ed Bartosh <ed.bartosh at linux.intel.com>
+ George Nita <george.nita at enea.com>
+ Maxin B. John <maxin.john at intel.com>
+ Paul Eggleton <paul.eggleton at linux.intel.com>
+ Qi.Chen at windriver.com <Qi.Chen at windriver.com>
+ Randy Witt <randy.e.witt at linux.intel.com>
+ Richard Purdie <richard.purdie at linuxfoundation.org>
+ Robert Yang <liezhi.yang at windriver.com>
+ Todor Minchev <todor.minchev at linux.intel.com>
+ Wenlin Kang <wenlin.kang at windriver.com>
License: GPL-2
Files: *
@@ -12,7 +23,7 @@ License: GPL-3
License: GPL-2
On Debian systems, the full text of the GNU General Public
- License version 3 can be found in the file
+ License version 2 can be found in the file
`/usr/share/common-licenses/GPL-2'.
License: GPL-3
diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py
index a044b951..1434c5e6 100644
--- a/elbepack/elbeproject.py
+++ b/elbepack/elbeproject.py
@@ -44,7 +44,7 @@ from elbepack.repomanager import ProjectRepo
from elbepack.config import cfg
from elbepack.pkgutils import extract_pkg
from elbepack.directories import (mako_template_dir, pack_dir)
-from elbepack.templates import template
+from elbepack.templates import template, write_pack_template
class IncompatibleArchitectureException(Exception):
@@ -65,6 +65,11 @@ class AptCacheCommitError(Exception):
Exception.__init__(self, "Error Committing rpcaptcache %s" % msg)
+class UnsupportedSDKException(Exception):
+ def __init__(self, triplet):
+ Exception.__init__(self, "SDK for %s currently unsupported" % triplet)
+
+
def test_gen_sdk_scripts():
os.system("mkdir -p /tmp/test/sdk")
gen_sdk_scripts('armhf-linux-gnueabihf',
@@ -90,20 +95,20 @@ def gen_sdk_scripts(triplet, prj_name, prj_version, builddir, sdkpath):
'sdk_title': 'ELBE %s' % prj_name,
'sdk_version': prj_version,
}
- t = os.path.join(mako_template_dir, 'toolchain-shar-extract.sh.mako')
sdkname = 'setup-elbe-sdk-%s-%s-%s.sh' % (triplet, prj_name, prj_version)
- with open(os.path.join(builddir, sdkname), 'w') as f:
- f.write(template(t, sdkvalues))
-
- t = os.path.join(mako_template_dir, 'environment-setup-elbe.mako')
+ write_pack_template(os.path.join(builddir, sdkname),
+ 'toolchain-shar-extract.sh.mako',
+ sdkvalues)
envname = 'environment-setup-elbe-%s-%s-%s' % (triplet,
prj_name,
prj_version)
- with open(os.path.join(sdkpath, envname), 'w') as f:
- f.write(template(t, sdkvalues))
+ write_pack_template(os.path.join(sdkpath, envname),
+ 'environment-setup-elbe.mako',
+ sdkvalues)
return sdkname
+
class ElbeProject (object):
def __init__(
self,
@@ -243,8 +248,8 @@ class ElbeProject (object):
raise AptCacheUpdateError(e)
try:
- self.get_rpcaptcache().mark_install_devpkgs(ignore_pkgs,
- ignore_dev_pkgs)
+ self.get_rpcaptcache().mark_install_devpkgs(set(ignore_pkgs),
+ set(ignore_dev_pkgs))
except SystemError as e:
self.log.printo("mark install devpkgs failed: %s" % str(e))
try:
@@ -279,6 +284,11 @@ class ElbeProject (object):
def build_sdk(self):
triplet = self.xml.defs["triplet"]
+ try:
+ crosstoolchainpkg = "gcc-%s" % self.xml.defs["sdkarch"]
+ except KeyError:
+ raise UnsupportedSDKException(triplet)
+
# build target sysroot including libs and headers for the target
self.build_sysroot()
sdktargetpath = os.path.join(self.sdkpath, "sysroots", "target")
@@ -286,7 +296,6 @@ class ElbeProject (object):
self.log.do("tar xJf %s/sysroot.tar.xz -C %s" % (self.builddir,
sdktargetpath))
# build host sysroot including cross compiler
- crosstoolchainpkg = "gcc-%s" % self.xml.defs["triplet"]
hostsysrootpath = os.path.join(self.sdkpath, 'sysroots', 'host')
self.log.do('mkdir -p "%s"' % hostsysrootpath)
extract_pkg(self.xml.prj,
diff --git a/elbepack/pkgutils.py b/elbepack/pkgutils.py
index 76dcc69d..e062396e 100644
--- a/elbepack/pkgutils.py
+++ b/elbepack/pkgutils.py
@@ -178,16 +178,20 @@ def get_dsc_size(fname):
return sz
-def download_pkg(prj, target_dir, defs, package, arch="default", incl_deps=False):
+def download_pkg(prj,
+ target_dir,
+ defs,
+ package,
+ arch="default",
+ incl_deps=False,
+ log=None):
try:
urilist = get_uri(prj, defs, arch, package, incl_deps)
except KeyError:
raise NoKinitrdException('no package %s available' % package)
- return
except SystemError:
raise NoKinitrdException('a configured mirror is not reachable')
- return
except CommandError as e:
raise NoKinitrdException("couldn't download package %s" % package)
@@ -216,24 +220,24 @@ def download_pkg(prj, target_dir, defs, package, arch="default", incl_deps=False
while len(buf) > 0:
m.update(buf)
buf = f.read(65536)
-
if m.hexdigest() != sha256:
raise NoKinitrdException('%s failed to verify !!!' % package)
else:
- # TODO this should be logged to validation.txt or stderr depending
- # on use-case
- print("-----------------------------------------------------")
- print("WARNING:")
- print("Using untrusted %s package" % package)
- print("-----------------------------------------------------")
+ if log:
+ log.printo("WARNING: Using untrusted %s package" % package)
+ else:
+ print("-----------------------------------------------------")
+ print("WARNING:")
+ print("Using untrusted %s package" % package)
+ print("-----------------------------------------------------")
return [y[0] for y in urilist]
def extract_pkg(prj, target_dir, defs, package, arch="default",
- incl_deps=False):
+ incl_deps=False, log=None):
- pkgs = download_pkg(prj, target_dir, defs, package, arch, incl_deps)
+ pkgs = download_pkg(prj, target_dir, defs, package, arch, incl_deps, log)
for package in pkgs:
ppath = os.path.join(target_dir, "%s.deb" % package)
diff --git a/elbepack/rfs.py b/elbepack/rfs.py
index 4916e063..6c2ffa4a 100644
--- a/elbepack/rfs.py
+++ b/elbepack/rfs.py
@@ -43,6 +43,9 @@ class BuildEnv ():
if clean:
self.rfs.rmtree("")
+ self.gen_base_files()
+
+ def gen_base_files(self):
# TODO think about reinitialization if elbe_version differs
if not self.rfs.isfile("etc/elbe_version"):
# avoid starting daemons inside the buildenv
@@ -102,6 +105,7 @@ class BuildEnv ():
# deboostraping causes errors if running on an existing filesystem
# ensure to run on an empty directory
self.rfs.rmtree("/")
+ self.gen_base_files()
primary_mirror = self.xml.get_primary_mirror(
self.rfs.fname('/cdrom/targetrepo'))
diff --git a/elbepack/rpcaptcache.py b/elbepack/rpcaptcache.py
index 975eec0f..7a90e838 100644
--- a/elbepack/rpcaptcache.py
+++ b/elbepack/rpcaptcache.py
@@ -112,16 +112,9 @@ class RPCAPTCache(InChRootObject):
from_user=from_user)
def mark_install_devpkgs(self, ignore_pkgs, ignore_dev_pkgs):
- ignore_pkgs.remove('libc6') # we don't want to ignore libc
- # we don't want to ignore libstdc++
- try:
- ignore_pkgs.remove('libstdc++5')
- except ValueError:
- pass
- try:
- ignore_pkgs.remove('libstdc++6')
- except ValueError:
- pass
+ ignore_pkgs.discard('libc6') # we don't want to ignore libc
+ ignore_pkgs.discard('libstdc++5')
+ ignore_pkgs.discard('libstdc++6')
# list all debian src packages of all installed packages that don't
# come from debootstrap
src_list = [
diff --git a/elbepack/xmldefaults.py b/elbepack/xmldefaults.py
index 2d357c54..4ac7de92 100644
--- a/elbepack/xmldefaults.py
+++ b/elbepack/xmldefaults.py
@@ -54,7 +54,8 @@ armhf_defaults = {
"console": "ttyAMA0,115200n1",
"machine": "versatilepb -cpu cortex-a9",
"nicmodel": "smc91c111",
- "triplet": "arm-linux-gnueabihf"
+ "triplet": "arm-linux-gnueabihf",
+ "sdkarch": "arm-linux-gnueabihf"
}
armhf_linaro48_defaults = {
@@ -138,7 +139,7 @@ i386_defaults = {
"console": "ttyS0,115200n1",
"machine": "pc",
"nicmodel": "virtio",
- "triplet": "i686-linux-gnu"
+ "triplet": "i386-linux-gnu"
}
archindep_defaults = {
--
2.15.1
More information about the elbe-devel
mailing list