[elbe-devel] [PATCH v2 4/5] Python3: harmonize binary/unicode strings

[Bastian Germann]

Am 13.12.19 um 12:45 schrieb Christian Teklenborg:
> Python 2 has binary strings as default whereas Python 3 has unicode strings.
> This requires changing the string handling at some places to explicitly use one
> or the other. This involves opening some files in binary mode and converting
> Popen streams to unicode strings.
> 
> The binascii.b2a_base64 call returns binary strings on Python 3 even though it
> could return unicode strings (base64 is ASCII). So we safely convert it via
> decode('ascii').
> 
> Signed-off-by: Christian Teklenborg <chris at linutronix.de>

Reviewed-by: Bastian Germann <bage at linutronix.de>

> ---
>  elbepack/daemons/soap/esoap.py |  8 ++++----
>  elbepack/kvm.py                | 10 ++++------
>  elbepack/log.py                |  5 ++++-
>  elbepack/shellhelper.py        |  6 ++++++
>  elbepack/soapclient.py         |  4 ++--
>  5 files changed, 20 insertions(+), 13 deletions(-)
> 
> diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py
> index 468df8a5..7d76d6a3 100644
> --- a/elbepack/daemons/soap/esoap.py
> +++ b/elbepack/daemons/soap/esoap.py
> @@ -159,7 +159,7 @@ class ESoap (ServiceBase):
>                  self.app.pm.set_current_project_xml(uid, fn)
>              return -2
>  
> -        with open(fn, 'a') as fp:
> +        with open(fn, 'ab') as fp:
>              fp.write(binascii.a2b_base64(blob))
>  
>          return part + 1
> @@ -264,7 +264,7 @@ class ESoap (ServiceBase):
>          cdrom_fname = os.path.join(builddir, "uploaded_cdrom.iso")
>  
>          # Now append data to cdrom_file
> -        fp = open(cdrom_fname, "a")
> +        fp = open(cdrom_fname, "ab")
>          fp.write(binascii.a2b_base64(data))
>          fp.close()
>  
> @@ -297,7 +297,7 @@ class ESoap (ServiceBase):
>          pdebuild_fname = os.path.join(builddir, "current_pdebuild.tar.gz")
>  
>          # Now write empty File
> -        fp = open(pdebuild_fname, "a")
> +        fp = open(pdebuild_fname, "ab")
>          fp.write(binascii.a2b_base64(data))
>          fp.close()
>  
> @@ -331,7 +331,7 @@ class ESoap (ServiceBase):
>          orig_fname = os.path.join(builddir, self.app.pm.get_orig_fname(uid))
>  
>          # Now append to File
> -        fp = open(orig_fname, "a")
> +        fp = open(orig_fname, "ab")
>          fp.write(binascii.a2b_base64(data))
>          fp.close()
>  
> diff --git a/elbepack/kvm.py b/elbepack/kvm.py
> index 62f2ddeb..8ea0e6dd 100644
> --- a/elbepack/kvm.py
> +++ b/elbepack/kvm.py
> @@ -6,7 +6,8 @@
>  # SPDX-License-Identifier: GPL-3.0-or-later
>  
>  import os
> -import subprocess
> +
> +from elbepack.shellhelper import command_out
>  
>  kvm_exe_list = [
>      '/usr/bin/kvm',
> @@ -21,11 +22,8 @@ def find_kvm_exe():
>      for fname in kvm_exe_list:
>          if os.path.isfile(fname) and os.access(fname, os.X_OK):
>              # determine kvm version
> -            cmd = subprocess.Popen(
> -                fname + ' --version',
> -                shell=True,
> -                stdout=subprocess.PIPE)
> -            for line in cmd.stdout:
> +            _, stdout = command_out(fname + ' --version')
> +            for line in stdout.splitlines():
>                  if "version" in line:
>                      version = line.split()[3].split('(')[0].strip()
>  
> diff --git a/elbepack/log.py b/elbepack/log.py
> index 43e04985..bffc6c1d 100644
> --- a/elbepack/log.py
> +++ b/elbepack/log.py
> @@ -11,6 +11,7 @@ import select
>  import threading
>  from contextlib import contextmanager
>  
> +from io import TextIOWrapper, BytesIO
>  
>  root = logging.getLogger()
>  root.setLevel(logging.DEBUG)
> @@ -241,7 +242,7 @@ class AsyncLogging(object):
>  
>      def run(self):
>          alive = True
> -        rest = ""
> +        rest = bytes()
>          while alive:
>              events = self.epoll.poll()
>              for _, event in events:
> @@ -256,6 +257,8 @@ class AsyncLogging(object):
>              if not rest:
>                  break
>  
> +        rest = TextIOWrapper(BytesIO(rest), encoding='utf-8', errors='replace').read()
> +
>          if self.lines:
>              self.lines[-1] += rest
>              self.block.info("\n".join(self.lines))
> diff --git a/elbepack/shellhelper.py b/elbepack/shellhelper.py
> index 0895c0b0..c73642c7 100644
> --- a/elbepack/shellhelper.py
> +++ b/elbepack/shellhelper.py
> @@ -12,6 +12,7 @@ from subprocess import Popen, PIPE, STDOUT, call
>  
>  from elbepack.log import async_logging
>  
> +from io import TextIOWrapper, BytesIO
>  
>  log = logging.getLogger("log")
>  soap = logging.getLogger("soap")
> @@ -54,6 +55,8 @@ def command_out(cmd, stdin=None, output=PIPE, env_add=None):
>                    stdout=output, stderr=STDOUT, stdin=PIPE, env=new_env)
>          out, _ = p.communicate(input=stdin)
>  
> +    out = TextIOWrapper(BytesIO(out), encoding='utf-8', errors='replace').read()
> +
>      return p.returncode, out
>  
>  
> @@ -81,6 +84,9 @@ def command_out_stderr(cmd, stdin=None, env_add=None):
>                    stdout=PIPE, stderr=PIPE, stdin=PIPE, env=new_env)
>          output, stderr = p.communicate(input=stdin)
>  
> +    output = TextIOWrapper(BytesIO(output), encoding='utf-8', errors='replace').read()
> +    stderr = TextIOWrapper(BytesIO(stderr), encoding='utf-8', errors='replace').read()
> +
>      return p.returncode, output, stderr
>  
>  
> diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
> index c654e410..cb8dbd8f 100644
> --- a/elbepack/soapclient.py
> +++ b/elbepack/soapclient.py
> @@ -87,7 +87,7 @@ class ElbeSoapClient(object):
>          self.service.login(user, passwd)
>  
>      def download_file(self, builddir, filename, dst_fname):
> -        fp = open(dst_fname, "w")
> +        fp = open(dst_fname, "wb")
>          part = 0
>  
>          # XXX the retry logic might get removed in the future, if the error
> @@ -327,7 +327,7 @@ class SetXmlAction(ClientAction):
>          part = 0
>          with open(filename, "rb") as fp:
>              while True:
> -                xml_base64 = binascii.b2a_base64(fp.read(size))
> +                xml_base64 = binascii.b2a_base64(fp.read(size)).decode('ascii')
>                  # finish upload
>                  if len(xml_base64) == 1:
>                      part = client.service.upload_file(builddir,
>