[elbe-devel] [PATCH v4 1/2] preprocess: allow building variants of a xml file

[Manuel Traut]

Other build-systems allow building different flavours or variants of a
image based on some config values. This eases the maintainance of very
similar images.

The elbe preprocess subcommand allows modifying XML files before they are
used with other elbe subcommands. The XML file used by the preprocess
subcommand doesn't need to validate with 'dbsfed.xsd', but the output needs
to.

This adds a new parameter '--variant' to the 'preprocess' subcommand. A
XML tag inside the XML file given to 'preprocess' can contain a 'variant'
attribute. If the value of the variant attribute inside XML matches with
a variant given as parameter to the preprocess command, the XML tag
stays inside the XML file. If the XML tag has a variant attribute but
doesn't match with the variant given as parameter to 'preprocess' the
XML tag will be dropped. If no 'variant' parameter is given to the
'preprocess' subcommand all tags with a 'variant' parameter are dropped.

This allows XML snippets like this:

  <url variant='security'>
          <binary>http://security.debian.org/ stretch/updates main</binary>
          <source>http://security.debian.org/ stretch/updates main</source>
  </url>

  <pkg-list>
          <pkg variant='security'>openssh-server</pkg>
          <pkg variant='audio,video'>totem</pkg>
  </pkg-list>

It is also possible to use multiple sections with variant attributes like this:

<pkg-list variant='audio>
  <pkg>alsa</pkg>
  <pkg>pavucontrol</pkg>
</pkg-list>
<pkg-list variant='video>
  <pkg>totem</pkg>
  <pkg>ffmpeg</pkg>
</pkg-list>

This needs defining mergeable sections by xpath.
A list of mergeable sections is hardcoded in elbepack/xmlpreprocess.py
If one of these sections occure multiple times the contents are merged into a
single section.

Signed-off-by: Manuel Traut <manut at linutronix.de>
---
 elbepack/commands/preprocess.py |  9 +++++-
 elbepack/xmlpreprocess.py       | 64 ++++++++++++++++++++++++++++++++++++++---
 2 files changed, 68 insertions(+), 5 deletions(-)

diff --git a/elbepack/commands/preprocess.py b/elbepack/commands/preprocess.py
index 63dd0295..7b2be1f9 100644
--- a/elbepack/commands/preprocess.py
+++ b/elbepack/commands/preprocess.py
@@ -29,6 +29,9 @@ def run_command(argv):
     oparser.add_option("-o", "--output", dest="output",
                        default="preprocess.xml",
                        help="preprocessed output file", metavar="<xmlfile>")
+    oparser.add_option("-v", "--variants", dest="variant",
+                       default=None,
+                       help="enable only tags with empty or given variant")
     (opt, args) = oparser.parse_args(argv)
 
     if len(args) != 1:
@@ -40,8 +43,12 @@ def run_command(argv):
         print("%s doesn't exist" % args[0], file=sys.stderr)
         sys.exit(20)
 
+    variants = []
+    if opt.variant:
+        variants = opt.variant.split(',')
+
     try:
-        xmlpreprocess(args[0], opt.output)
+        xmlpreprocess(args[0], opt.output, variants)
     except XMLPreprocessError as e:
         print(e, file=sys.stderr)
         sys.exit(20)
diff --git a/elbepack/xmlpreprocess.py b/elbepack/xmlpreprocess.py
index 86eec9a6..d24f0801 100644
--- a/elbepack/xmlpreprocess.py
+++ b/elbepack/xmlpreprocess.py
@@ -17,16 +17,26 @@
 # along with ELBE.  If not, see <http://www.gnu.org/licenses/>.
 
 import sys
+import re
 from lxml import etree
 from lxml.etree import XMLParser, parse
 
+# list of sections that are allowed to exists multiple times before
+# preprocess and that childrens are merge into one section during preprocess
+mergepaths = ['//target/finetuning',
+              '//target/pkg-list',
+              '//project/buildimage/pkg-list']
+
 
 class XMLPreprocessError(Exception):
-    def __init__(self, message):
-        Exception.__init__(self, message)
+    pass
+
+
+def xmlpreprocess(fname, output, variants=[]):
 
+    # first convert variants to a set
+    variants = set(variants)
 
-def xmlpreprocess(fname, output):
     schema_file = "https://www.linutronix.de/projects/Elbe/dbsfed.xsd"
     parser = XMLParser(huge_tree=True)
     schema_tree = etree.parse(schema_file)
@@ -36,18 +46,64 @@ def xmlpreprocess(fname, output):
         xml = parse(fname, parser=parser)
         xml.xinclude()
 
+        # Variant management
+        # check all nodes for variant field, and act accordingly.
+        # The result will not contain any variant attributes anymore.
+        rmlist = []
+        for tag in xml.iter('*'):
+            if 'variant' in tag.attrib:
+                tag_variants = set (tag.attrib['variant'].split(','))
+
+                # check if tag_variants intersects with
+                # active variants.
+                intersect = variants.intersection(tag_variants)
+
+                if len(intersect):
+                    # variant is wanted, keep it and remove the variant
+                    # attribute
+                    tag.attrib.pop('variant')
+                else:
+                    # tag has a variant attribute but the variant was not
+                    # specified: remove the tag delayed
+                    rmlist.append(tag)
+
+        for tag in rmlist:
+            tag.getparent().remove(tag)
+
+        # if there are multiple sections because of sth like '<finetuning
+        # variant='A'> ...  and <finetuning variant='B'> and running preprocess
+        # with --variant=A,B the two sections need to be merged
+        #
+        # Use xpath expressions to identify mergeable sections.
+        for mergepath in mergepaths:
+            mergenodes = xml.xpath (mergepath)
+
+            # if there is just one section of a type
+            # or no section, nothing needs to be done
+            if len(mergenodes) < 2:
+                continue
+
+            # append all childrens of section[1..n] to section[0] and delete
+            # section[1..n]
+            for section in mergenodes[1:]:
+                for c in section.getchildren():
+                    mergenodes[0].append (c)
+                section.getparent().remove(section)
+
         if schema.validate(xml):
+            # if validation succedes write xml file
             xml.write(
                 output,
                 encoding="UTF-8",
                 pretty_print=True,
                 compression=9)
+            # the rest of the code is exception and error handling
             return
 
     except etree.XMLSyntaxError:
         raise XMLPreprocessError("XML Parse error\n" + str(sys.exc_info()[1]))
     except BaseException:
-        XMLPreprocessError(
+        raise XMLPreprocessError(
             "Unknown Exception during validation\n" + str(sys.exc_info()[1]))
 
     # We have errors, return them in string form...
-- 
2.15.1