[elbe-devel] [PATCH 1/3] cd2aptly: initial version
Torben Hohn
torben.hohn at linutronix.de
Fri Sep 21 11:03:52 CEST 2018
On Fri, Sep 14, 2018 at 03:06:23PM +0200, Manuel Traut wrote:
> cd2aptly can be used to add bin-cdrom.iso and source-cdrom isos
> to an aptly repository. Goal is to be able to rebuild a XML with
> a so created repository with no online connection.
this needs an explanation, why the http server is used.
etc.
>
> Signed-off-by: Manuel Traut <manut at linutronix.de>
> ---
> debian/control | 17 +++
> debian/python-elbe-cd2aptly.install | 1 +
> debian/python3-elbe-cd2aptly.install | 1 +
> elbepack/commands/cd2aptly.py | 196 +++++++++++++++++++++++++++
> 4 files changed, 215 insertions(+)
> create mode 100644 debian/python-elbe-cd2aptly.install
> create mode 100644 debian/python3-elbe-cd2aptly.install
> create mode 100644 elbepack/commands/cd2aptly.py
>
> [...]
>
> Package: elbe-updated
> Architecture: all
> diff --git a/debian/python-elbe-cd2aptly.install b/debian/python-elbe-cd2aptly.install
> new file mode 100644
> index 00000000..a756d02d
> --- /dev/null
> +++ b/debian/python-elbe-cd2aptly.install
> @@ -0,0 +1 @@
> +./usr/lib/python2.*/*-packages/elbepack/commands/cd2aptly.py
> diff --git a/debian/python3-elbe-cd2aptly.install b/debian/python3-elbe-cd2aptly.install
> new file mode 100644
> index 00000000..10a45eee
> --- /dev/null
> +++ b/debian/python3-elbe-cd2aptly.install
> @@ -0,0 +1 @@
> +./usr/lib/python3.*/*-packages/elbepack/commands/cd2aptly.py
> diff --git a/elbepack/commands/cd2aptly.py b/elbepack/commands/cd2aptly.py
> new file mode 100644
> index 00000000..f1404751
> --- /dev/null
> +++ b/elbepack/commands/cd2aptly.py
> @@ -0,0 +1,196 @@
> +# ELBE - Debian Based Embedded Rootfilesystem Builder
> +# Copyright (c) 2018 Manuel Traut <manut at linutronix.de>
> +#
> +# SPDX-License-Identifier: GPL-3.0-or-later
> +
> +from __future__ import print_function
> +
> +import sys
> +import os
> +import tempfile
> +import SimpleHTTPServer
> +import SocketServer
> +
> +from threading import (Thread, Condition)
> +from optparse import (OptionParser)
> +from subprocess import (check_call, CalledProcessError)
> +
> +def _check_call(cmd, shell=False):
> + if shell:
> + print("SH: %s" % cmd)
> + else:
> + print("CMD: %s" % cmd)
> + check_call(cmd, shell=shell)
i believe, that we should continue to use the stuff from
elbepack/shellhelper.py
at least until we decide, to replace all shellhelper.system()
with check_call()
> +
> +class RepoHost(object):
> + def __init__ (self, path):
> + self.path = path
> + self.httpd = None
> + self.thread = None
> + self.port = None
> + self.ready = Condition()
> +
> + def __enter__ (self):
> + self.thread = Thread(target=self.serve)
> + self.ready.acquire()
> + self.thread.start()
> + self.ready.wait()
> + return self.port
> +
> + def __exit__ (self, t, value, traceback):
> + if self.httpd:
> + self.httpd.shutdown()
> + else:
> + print("no httpd running, nothing to shut down", file=sys.stderr)
> + self.thread.join()
> + print("shutdown %d DONE" % self.port)
> +
> + def serve(self):
> + self.ready.acquire()
> + handler = SimpleHTTPServer.SimpleHTTPRequestHandler
> + os.chdir(self.path)
this is not so good. its process wide.
https://stackoverflow.com/questions/16388400/what-is-a-thread-specific-os-chdir-and-mkdir-in-python
another reason, why i dont think, we should be using SimpleHTTPServer.
> + self.httpd = SocketServer.TCPServer(('', 0), handler)
i bet, this opens a server thats not only on localhost.
probably good: SocketServer.TCPServer(('localhost', 0)
> + self.port = self.httpd.server_address[1]
> + self.ready.notify()
> + self.ready.release()
> + self.httpd.serve_forever()
> + os.chdir('/')
> + print("shutdown: %d" % self.port)
look here... the condition is not necessary:
----------------------------------------------------------------------------
import threading
import SocketServer
import SimpleHTTPServer
class ThreadedHTTPServer(object):
handler = SimpleHTTPServer.SimpleHTTPRequestHandler
def __init__(self, host, port):
self.server = SocketServer.TCPServer((host, port), self.handler)
self.server_thread = threading.Thread(target=self.server.serve_forever)
self.server_thread.daemon = True
def start(self):
self.server_thread.start()
def stop(self):
self.server.shutdown()
self.server.server_close()
----------------------------------------------------------------------------
just make the thread run self.server.serve_forever, and
create the server in __enter__ and also obtain the port....
i am not very fond of using yet another webserver (SimpleHTTPServer)
but dont see an easy way to use cherrypy for this.
> +
> +def snapshot_from_repo(repo, mirrorname, port, src_cd_support, cmd='aptly'):
> + section = 'main'
> + names = []
> + if not os.path.exists(repo):
> + return names
> + for release in os.listdir(os.path.join(repo, 'dists')):
> + name = mirrorname + '-' + release
> + host = "http://localhost:%d" % port
> +
> + if 'target' in repo:
> + name = name + '-target'
> + host = "http://localhost:%d/targetrepo" % port
> +
> + names.append(name)
> +
> + wb = "-with-udebs"
> + if os.path.exists(os.path.join(repo, 'dists', release, 'main/source')):
> + if not src_cd_support:
> + print("aptly lacks src cd support, cd cant be handled",
> + file=sys.stderr)
> + else:
> + wb="-with-binaries=false -with-sources=true"
> +
> + if os.path.exists(os.path.join(repo, 'dists', release, 'added')):
> + section = section + ' added'
> +
> + _check_call("%s mirror create %s %s %s %s %s" % (cmd, wb, name, host,
> + release, section),
> + shell=True)
> +
> + _check_call("%s mirror update %s" % (cmd, name), shell=True)
> + _check_call("%s snapshot create %s from mirror %s" % (cmd, name, name),
> + shell=True)
> +
> + return names
> +
> +def run_command(argv):
> + usage = "usage: elbe cd2aptly <cdrom.iso> <aptly-repo-path>"
> + oparser = OptionParser(usage=usage)
> + (_opt, args) = oparser.parse_args(argv)
> +
> + if len(args) < 2:
> + print(usage, file=sys.stderr)
> + return
> +
> + # detect aptly feature to work with src-repo only
> + src_cd_support = True
> + try:
> + _check_call("aptly mirror create 2>&1 | grep \"with-binaries\"",
> + shell=True)
> + except CalledProcessError as e:
> + print(e, file=sys.stderr)
> + print("src_cd_support = false", file=sys.stderr)
> + src_cd_support = False
> +
> + mntdir = tempfile.mkdtemp()
please use TmpdirFilesystem
tmpdir is not cleaned up in current code.
> +
> + dists = {'jessie': [], 'wheezy': [], 'stretch': [], 'buster': [], 'sid': []}
> + isoimage = argv[0]
> + aptlyrepo = os.path.abspath(argv[1])
> + aptlyconf = os.path.join(aptlyrepo, 'aptly.conf')
> + namesfile = os.path.join(aptlyrepo, 'snapshots')
> + mainrepo = mntdir
> + targetrepo = os.path.join(mntdir, 'targetrepo')
> + mirrorname = mntdir.split('/')[-1:][0]
> + names = []
> +
> + cmd = "aptly -config=%s " % aptlyconf
> +
> + _check_call(["mkdir", "-p", mntdir])
> + _check_call(["mkdir", "-p", aptlyrepo])
> +
> + with open(aptlyconf, 'w') as ac:
> + ac.write("{\n")
> + ac.write(" \"rootDir\": \"%s\",\n" % aptlyrepo)
> + ac.write(" \"gpgDisableVerify\": true,\n")
> + ac.write(" \"gpgDisableSign\": true\n")
> + ac.write("}\n")
> +
> + _check_call(["fuseiso9660", isoimage, mntdir])
> +
> + with RepoHost(mntdir) as port:
> +
> + if os.path.exists(namesfile):
> + with open(namesfile, 'r') as nf:
> + for n in nf:
> + names.append(n)
> +
> + names += snapshot_from_repo(mainrepo,
> + mirrorname,
> + port,
> + src_cd_support,
> + cmd)
> +
> + names += snapshot_from_repo(targetrepo,
> + mirrorname,
> + port,
> + src_cd_support,
> + cmd)
> +
> + with open(namesfile, 'w') as nf:
> + for n in names:
> + nf.write(n+'\n')
> +
this is all a bit unclear, what it does.
some comments would be nice.
> + for r,rl in dists.items():
> + for n in names:
> + if r in n:
> + rl.append(n)
> +
> + for r,rl in dists.items():
> + try:
> + _check_call("%s publish drop %s" % (cmd, r), shell=True)
> + except CalledProcessError as e:
> + print(e)
> + try:
> + _check_call("%s snapshot drop --force repo-%s" % (cmd, r),
> + shell=True)
> + except CalledProcessError as e:
> + print(e)
> +
> + n = ''
> + for sn in rl:
> + n = n + ' ' + sn
n = ' '.join(rl)
> +
> + n = n.replace('\n', ' ')
> + if n:
> + _check_call("%s snapshot merge -no-remove repo-%s %s" % (cmd, r, n),
> + shell=True)
> +
> + _check_call("%s publish snapshot -distribution=\"%s\" repo-%s" % (cmd,
> + r,
> + r),
> + shell=True)
> +
> + _check_call(["fusermount", "-u", mntdir])
> + _check_call(["rm", "-rf", mntdir])
> + print(dists)
> --
> 2.19.0.rc2
>
--
Torben Hohn
Linutronix GmbH | Bahnhofstrasse 3 | D-88690 Uhldingen-Mühlhofen
Phone: +49 7556 25 999 18; Fax.: +49 7556 25 999 99
Hinweise zum Datenschutz finden Sie hier (Informations on data privacy
can be found here): https://linutronix.de/kontakt/Datenschutz.php
Linutronix GmbH | Firmensitz (Registered Office): Uhldingen-Mühlhofen |
Registergericht (Registration Court): Amtsgericht Freiburg i.Br., HRB700
806 | Geschäftsführer (Managing Directors): Heinz Egger, Thomas Gleixner
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: not available
URL: <http://lists.linutronix.de/pipermail/elbe-devel/attachments/20180921/8dcb8c7e/attachment.sig>
More information about the elbe-devel
mailing list