[elbe-devel] [PATCH v3 28/52] Async logging for rpcaptcache

Torben Hohn torben.hohn at linutronix.de
Fri Jul 5 13:01:48 CEST 2019


On Thu, Jun 27, 2019 at 02:44:42PM +0200, dion at linutronix.de wrote:
> From: Olivier Dion <dion at linutronix.de>
> 
> Before the RPC is started, its stdout and stderr are redirect to an
> async logging thread that will ensure that _all_ outputs from the RPC
> and the RPC children are properly catch.  See async_logging for more
> details.
> 
> It's thus ok to use /print/ statements _only_ in the methods of
> RPCAPTCache since everything is redirect to the logging thread.
> 
> Signed-off-by: Olivier Dion <dion at linutronix.de>

Reviewed-by: Torben Hohn <torben.hohn at linutronix.de>

> ---
>  elbepack/cdroms.py      |  6 ++--
>  elbepack/elbeproject.py | 11 +++----
>  elbepack/rpcaptcache.py | 79 ++++++++++++++++++++++++++++++-------------------
>  3 files changed, 56 insertions(+), 40 deletions(-)
> 
> diff --git a/elbepack/cdroms.py b/elbepack/cdroms.py
> index 3f2ecc21..e16dcd2d 100644
> --- a/elbepack/cdroms.py
> +++ b/elbepack/cdroms.py
> @@ -49,7 +49,7 @@ def mk_source_cdrom(
>                          cdrom_size,
>                          mirror)
>  
> -    cache = get_rpcaptcache(rfs, "aptcache.log", arch)
> +    cache = get_rpcaptcache(rfs, arch)
>      cache.update()
>      pkglist = cache.get_installed_pkgs()
>  
> @@ -150,7 +150,7 @@ def mk_binary_cdrom(
>                                 target_repo_path, log, cdrom_size, mirror)
>  
>      if xml is not None:
> -        cache = get_rpcaptcache(rfs, "aptcache.log", arch)
> +        cache = get_rpcaptcache(rfs, arch)
>          for p in xml.node("debootstrappkgs"):
>              pkg = XMLPackage(p, arch)
>              try:
> @@ -179,7 +179,7 @@ def mk_binary_cdrom(
>                      pkg.installed_version +
>                      " missing name or version")
>  
> -    cache = get_rpcaptcache(rfs, "aptcache.log", arch)
> +    cache = get_rpcaptcache(rfs, arch)
>      pkglist = cache.get_installed_pkgs()
>      for pkg in pkglist:
>          try:
> diff --git a/elbepack/elbeproject.py b/elbepack/elbeproject.py
> index 073bfb15..6911a9c9 100644
> --- a/elbepack/elbeproject.py
> +++ b/elbepack/elbeproject.py
> @@ -807,13 +807,10 @@ class ElbeProject (object):
>              arch = env.arch
>  
>          if env.rpcaptcache is None:
> -            env.rpcaptcache = get_rpcaptcache(
> -                env.rfs,
> -                self.log.fp.name,
> -                arch,
> -                self.rpcaptcache_notifier,
> -                norecommend,
> -                self.xml.prj.has('noauth'))
> +            env.rpcaptcache = get_rpcaptcache(env.rfs, arch,
> +                                              self.rpcaptcache_notifier,
> +                                              norecommend,
> +                                              self.xml.prj.has('noauth'))
>          return env.rpcaptcache
>  
>      def drop_rpcaptcache(self, env=None):
> diff --git a/elbepack/rpcaptcache.py b/elbepack/rpcaptcache.py
> index d4f4a794..e332dd03 100644
> --- a/elbepack/rpcaptcache.py
> +++ b/elbepack/rpcaptcache.py
> @@ -6,6 +6,9 @@
>  #
>  # SPDX-License-Identifier: GPL-3.0-or-later
>  
> +from __future__ import print_function
> +
> +import logging
>  import os
>  import sys
>  import time
> @@ -20,6 +23,42 @@ from apt import Cache
>  from elbepack.aptprogress import (ElbeAcquireProgress, ElbeInstallProgress,
>                                    ElbeOpProgress)
>  from elbepack.aptpkgutils import getalldeps, APTPackage
> +from elbepack.log import async_logging
> +
> +
> +log = logging.getLogger("log")
> +soap = logging.getLogger("soap")
> +
> +
> +class MyMan(BaseManager):
> +
> +    @staticmethod
> +    def register(typeid):
> +        """Register to BaseManager through decorator"""
> +        def _register(cls):
> +            BaseManager.register(typeid, cls)
> +            return cls
> +        return _register
> +
> +    @staticmethod
> +    def redirect_outputs(r, w):
> +        """Redirect all outputs to the writing end of a pipe 'w'"""
> +        os.close(r)
> +        os.dup2(w, os.sys.stdout.fileno())
> +        os.dup2(w, os.sys.stderr.fileno())
> +        # Buffering of 1 because in Python3 buffering of 0 is illegal
> +        # for non binary mode ..
> +        os.sys.stdout = os.fdopen(os.sys.stdout.fileno(), "w", 1)
> +        os.sys.stderr = os.fdopen(os.sys.stderr.fileno(), "w", 1)
> +        os.sys.__stdout__ = os.sys.stdout
> +        os.sys.__stderr__ = os.sys.stderr
> +
> +    def start(self):
> +        """Redirect outputs of the process to an async logging thread"""
> +        r, w = os.pipe()
> +        super(MyMan, self).start(MyMan.redirect_outputs, [r, w])
> +        async_logging(r, w, soap, log)
> +
>  
>  class InChRootObject(object):
>      def __init__(self, rfs):
> @@ -28,23 +67,14 @@ class InChRootObject(object):
>          self.finalizer = Finalize(self, self.rfs.leave_chroot, exitpriority=10)
>  
>  
> + at MyMan.register("RPCAPTCache")
>  class RPCAPTCache(InChRootObject):
> +
>      # pylint: disable=too-many-public-methods
> -    def __init__(
> -            self,
> -            rfs,
> -            log,
> -            arch,
> -            notifier=None,
> -            norecommend=False,
> -            noauth=True):
> +    def __init__(self, rfs, arch,
> +                 notifier=None, norecommend=False, noauth=True):
>  
>          # pylint: disable=too-many-arguments
> -
> -        sys.stdout = open(log, 'a', buffering=0)
> -        sys.stderr = open(log, 'a', buffering=0)
> -        self.logfile = open(log, 'a', buffering=0)
> -
>          InChRootObject.__init__(self, rfs)
>  
>          self.notifier = notifier
> @@ -99,6 +129,7 @@ class RPCAPTCache(InChRootObject):
>          return ret
>  
>      def mark_install(self, pkgname, version, from_user=True, nodeps=False):
> +        print('Mark for install "%s"' % pkgname)
>          p = self.cache[pkgname]
>          if version:
>              p.candidate = p.versions[version]
> @@ -165,8 +196,9 @@ class RPCAPTCache(InChRootObject):
>      def commit(self):
>          os.environ["DEBIAN_FRONTEND"] = "noninteractive"
>          os.environ["DEBONF_NONINTERACTIVE_SEEN"] = "true"
> +        print("Commiting changes ...")
>          self.cache.commit(ElbeAcquireProgress(),
> -                          ElbeInstallProgress(fileno=self.logfile.fileno()))
> +                          ElbeInstallProgress(fileno=sys.stdout.fileno()))
>          self.cache.open(progress=ElbeOpProgress())
>  
>      def clear(self):
> @@ -275,21 +307,8 @@ class RPCAPTCache(InChRootObject):
>                                                 unpack=False)
>              return self.rfs.fname(rel_filename)
>  
> -
> -class MyMan(BaseManager):
> -    pass
> -
> -
> -MyMan.register("RPCAPTCache", RPCAPTCache)
> -
> -
> -def get_rpcaptcache(
> -        rfs,
> -        log,
> -        arch,
> -        notifier=None,
> -        norecommend=False,
> -        noauth=True):
> +def get_rpcaptcache(rfs, arch,
> +                    notifier=None, norecommend=False, noauth=True):
>  
>      # pylint: disable=too-many-arguments
>  
> @@ -301,4 +320,4 @@ def get_rpcaptcache(
>      # MyMan.register()
>      #
>      # pylint: disable=no-member
> -    return mm.RPCAPTCache(rfs, log, arch, notifier, norecommend, noauth)
> +    return mm.RPCAPTCache(rfs, arch, notifier, norecommend, noauth)
> -- 
> 2.11.0
> 
> 
> _______________________________________________
> elbe-devel mailing list
> elbe-devel at linutronix.de
> https://lists.linutronix.de/mailman/listinfo/elbe-devel

-- 
Torben Hohn
Linutronix GmbH | Bahnhofstrasse 3 | D-88690 Uhldingen-Mühlhofen
Phone: +49 7556 25 999 18; Fax.: +49 7556 25 999 99

Hinweise zum Datenschutz finden Sie hier (Informations on data privacy 
can be found here): https://linutronix.de/kontakt/Datenschutz.php

Linutronix GmbH | Firmensitz (Registered Office): Uhldingen-Mühlhofen | 
Registergericht (Registration Court): Amtsgericht Freiburg i.Br., HRB700 
806 | Geschäftsführer (Managing Directors): Heinz Egger, Thomas Gleixner



More information about the elbe-devel mailing list