[elbe-devel] [RFC] cd2aptly: initial version

Manuel Traut manut at linutronix.de
Mon Jul 23 15:45:20 CEST 2018


Hi,

this is an early post, just to get a better overview about the goals of the
patches posted these days.

cd2aptly can be used to add bin-cdrom.iso and source-cdrom isos
to an aptly repository. Goal is to be able to rebuild a XML with
a so created repository with no online connection and no ISO.

A patched version of aptly is needed that allows importing src only
repositories.

knwon-issues:
 * if rebuilding from such a repo, the src-cdrom.iso cant be created for
    unknown reasons

---
 debian/control                       |  17 +++
 debian/python-elbe-cd2aptly.install  |   1 +
 debian/python3-elbe-cd2aptly.install |   1 +
 elbepack/commands/cd2aptly.py        | 198 +++++++++++++++++++++++++++
 4 files changed, 217 insertions(+)
 create mode 100644 debian/python-elbe-cd2aptly.install
 create mode 100644 debian/python3-elbe-cd2aptly.install
 create mode 100644 elbepack/commands/cd2aptly.py

diff --git a/debian/control b/debian/control
index 158df1ba..cd94db7c 100644
--- a/debian/control
+++ b/debian/control
@@ -241,6 +241,23 @@ Depends: ${misc:Depends},
 Description: Commandline Tool to control an elbe buildenv
  The SOAP Interface is used to control an elbe buildenv.
 
+Package: python-elbe-cd2aptly
+Section: text
+Architecture: all
+Depends: ${misc:Depends}, fuseiso9660, aptly
+Description: insert packages from elbe generated cdroms into aptly repos
+ iso images generated by elbe builds can be added to a aptly repository to
+ use local mirrors without internet connection for rebuilding the same image
+ again
+
+Package: python3-elbe-cd2aptly
+Section: text
+Architecture: all
+Depends: ${misc:Depends}, fuseiso9660, aptly
+Description: insert packages from elbe generated cdroms into aptly repos
+ iso images generated by elbe builds can be added to a aptly repository to
+ use local mirrors without internet connection for rebuilding the same image
+ again
 
 Package: elbe-updated
 Architecture: all
diff --git a/debian/python-elbe-cd2aptly.install b/debian/python-elbe-cd2aptly.install
new file mode 100644
index 00000000..a756d02d
--- /dev/null
+++ b/debian/python-elbe-cd2aptly.install
@@ -0,0 +1 @@
+./usr/lib/python2.*/*-packages/elbepack/commands/cd2aptly.py
diff --git a/debian/python3-elbe-cd2aptly.install b/debian/python3-elbe-cd2aptly.install
new file mode 100644
index 00000000..10a45eee
--- /dev/null
+++ b/debian/python3-elbe-cd2aptly.install
@@ -0,0 +1 @@
+./usr/lib/python3.*/*-packages/elbepack/commands/cd2aptly.py
diff --git a/elbepack/commands/cd2aptly.py b/elbepack/commands/cd2aptly.py
new file mode 100644
index 00000000..1cf9d40b
--- /dev/null
+++ b/elbepack/commands/cd2aptly.py
@@ -0,0 +1,198 @@
+# ELBE - Debian Based Embedded Rootfilesystem Builder
+# Copyright (c) 2018 Manuel Traut <manut at linutronix.de>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import print_function
+
+import sys
+import os
+import tempfile
+import SimpleHTTPServer
+import SocketServer
+
+from threading import (Thread, Condition)
+from optparse import (OptionParser, OptionGroup)
+from subprocess import (check_call, CalledProcessError)
+from time import sleep
+
+def _check_call(cmd, shell=False):
+    if shell:
+        print("SH: %s" % cmd)
+    else:
+        print("CMD: %s" % cmd)
+    check_call(cmd, shell=shell)
+
+class RepoHost():
+    def __init__ (self, path):
+        self.path = path
+        self.httpd = None
+        self.ready = Condition()
+
+    def __enter__ (self):
+        self.thread = Thread(target=self.serve)
+        self.ready.acquire()
+        self.thread.start()
+        self.ready.wait()
+        return self.port
+
+    def __exit__ (self, type, value, traceback):
+        if self.httpd:
+            self.httpd.shutdown()
+        else:
+            print("no httpd running, nothing to shut down", file=sys.stderr)
+        self.thread.join()
+        print("shutdown %d DONE" % self.port)
+
+    def serve(self):
+        self.ready.acquire()
+        handler = SimpleHTTPServer.SimpleHTTPRequestHandler
+        os.chdir(self.path)
+        self.httpd = SocketServer.TCPServer(('', 0), handler)
+        self.port = self.httpd.server_address[1]
+        self.ready.notify()
+        self.ready.release()
+        self.httpd.serve_forever()
+        os.chdir('/')
+        print("shutdown: %d" % self.port)
+
+def snapshot_from_repo(repo, mirrorname, port, src_cd_support, cmd='aptly'):
+    section = 'main'
+    names = []
+    if not os.path.exists(repo):
+        return names
+    for release in os.listdir(os.path.join(repo, 'dists')):
+        name = mirrorname + '-' +  release
+        host = "http://localhost:%d" % port
+
+        if ('target' in repo):
+            name = name + '-target'
+            host = "http://localhost:%d/targetrepo" % port
+
+        names.append(name)
+
+        wb = "-with-udebs"
+        if os.path.exists(os.path.join(repo, 'dists', release, 'main/source')):
+            if not src_cd_support:
+                print("aptly lacks src cd support, cd cant be handled",
+                      file=sys.stderr)
+            else:
+                wb="-with-binaries=false -with-sources=true"
+
+        if os.path.exists(os.path.join(repo, 'dists', release, 'added')):
+            section = section + ' added'
+
+        _check_call("%s mirror create %s %s %s %s %s" % (cmd, wb, name, host,
+                                                         release, section),
+                    shell=True)
+
+        _check_call("%s mirror update %s" % (cmd, name), shell=True)
+        _check_call("%s snapshot create %s from mirror %s" % (cmd, name, name),
+                    shell=True)
+
+    return names
+
+def run_command(argv):
+    usage = "usage: elbe cd2aptly <cdrom.iso> <aptly-repo-path>"
+    oparser = OptionParser(usage=usage)
+    (opt, args) = oparser.parse_args(argv)
+
+    if len(args) < 2:
+        print(usage, file=sys.stderr)
+        return
+
+    # detect aptly feature to work with src-repo only
+    src_cd_support = True
+    try:
+        _check_call("aptly mirror create 2>&1 | grep \"with-binaries\"",
+                   shell=True)
+    except CalledProcessError as e:
+        print(e, file=sys.stderr)
+        print("src_cd_support = false", file=sys.stderr)
+        src_cd_support = False
+
+    mntdir = tempfile.mkdtemp()
+
+    dists = {'jessie': [], 'wheezy': [], 'stretch': [], 'buster': [], 'sid': []}
+    isoimage = argv[0]
+    aptlyrepo = os.path.abspath(argv[1])
+    aptlyconf = os.path.join(aptlyrepo, 'aptly.conf')
+    namesfile = os.path.join(aptlyrepo, 'snapshots')
+    mainrepo = mntdir
+    targetrepo = os.path.join(mntdir, 'targetrepo')
+    mirrorname = mntdir.split('/')[-1:][0]
+    names = []
+
+    cmd = "aptly -config=%s " % aptlyconf
+
+    _check_call(["mkdir", "-p", mntdir])
+    _check_call(["mkdir", "-p", aptlyrepo])
+
+    with open(aptlyconf, 'w') as ac:
+        ac.write("{\n")
+        ac.write("    \"rootDir\": \"%s\",\n" % aptlyrepo)
+        ac.write("    \"gpgDisableVerify\": true,\n")
+        ac.write("    \"gpgDisableSign\": true\n")
+        ac.write("}\n")
+
+    _check_call(["fuseiso9660", isoimage, mntdir])
+
+    with RepoHost(mntdir) as port:
+
+        if os.path.exists(namesfile):
+            with open(namesfile, 'r') as nf:
+                for n in nf:
+                    names.append(n)
+
+        names += snapshot_from_repo(mainrepo,
+                                    mirrorname,
+                                    port,
+                                    src_cd_support,
+                                    cmd)
+
+        names += snapshot_from_repo(targetrepo,
+                                    mirrorname,
+                                    port,
+                                    src_cd_support,
+                                    cmd)
+
+        with open(namesfile, 'w') as nf:
+            for n in names:
+                nf.write(n+'\n')
+
+        for r,rl in dists.items():
+            for n in names:
+                if r in n:
+                    rl.append(n)
+
+        for r,rl in dists.items():
+            try:
+                _check_call("%s publish drop %s" % (cmd, r), shell=True)
+            except CalledProcessError as e:
+                print(e)
+                pass
+            try:
+                _check_call("%s snapshot drop --force repo-%s" % (cmd, r),
+                            shell=True)
+            except CalledProcessError as e:
+                print(e)
+                pass
+
+            n = ''
+            for sn in rl:
+                n = n + ' ' + sn
+
+            n = n.replace('\n', ' ')
+            if n:
+                _check_call("%s snapshot merge -no-remove repo-%s %s" % (cmd, r, n),
+                            shell=True)
+
+                _check_call("%s publish snapshot -distribution=\"%s\" repo-%s" % (cmd,
+                                                                                r,
+                                                                                r),
+                            shell=True)
+
+    _check_call(["fusermount", "-u", mntdir])
+    _check_call(["rm", "-rf", mntdir])
+    print(dists)
+
-- 
2.18.0




More information about the elbe-devel mailing list