[elbe-devel] [PATCH v2 4/5] Python3: harmonize binary/unicode strings

[Christian Teklenborg]

Python 2 has binary strings as default whereas Python 3 has unicode strings.
This requires changing the string handling at some places to explicitly use one
or the other. This involves opening some files in binary mode and converting
Popen streams to unicode strings.

The binascii.b2a_base64 call returns binary strings on Python 3 even though it
could return unicode strings (base64 is ASCII). So we safely convert it via
decode('ascii').

Signed-off-by: Christian Teklenborg <chris at linutronix.de>
---
 elbepack/daemons/soap/esoap.py |  8 ++++----
 elbepack/kvm.py                | 10 ++++------
 elbepack/log.py                |  5 ++++-
 elbepack/shellhelper.py        |  6 ++++++
 elbepack/soapclient.py         |  4 ++--
 5 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py
index 468df8a5..7d76d6a3 100644
--- a/elbepack/daemons/soap/esoap.py
+++ b/elbepack/daemons/soap/esoap.py
@@ -159,7 +159,7 @@ class ESoap (ServiceBase):
                 self.app.pm.set_current_project_xml(uid, fn)
             return -2
 
-        with open(fn, 'a') as fp:
+        with open(fn, 'ab') as fp:
             fp.write(binascii.a2b_base64(blob))
 
         return part + 1
@@ -264,7 +264,7 @@ class ESoap (ServiceBase):
         cdrom_fname = os.path.join(builddir, "uploaded_cdrom.iso")
 
         # Now append data to cdrom_file
-        fp = open(cdrom_fname, "a")
+        fp = open(cdrom_fname, "ab")
         fp.write(binascii.a2b_base64(data))
         fp.close()
 
@@ -297,7 +297,7 @@ class ESoap (ServiceBase):
         pdebuild_fname = os.path.join(builddir, "current_pdebuild.tar.gz")
 
         # Now write empty File
-        fp = open(pdebuild_fname, "a")
+        fp = open(pdebuild_fname, "ab")
         fp.write(binascii.a2b_base64(data))
         fp.close()
 
@@ -331,7 +331,7 @@ class ESoap (ServiceBase):
         orig_fname = os.path.join(builddir, self.app.pm.get_orig_fname(uid))
 
         # Now append to File
-        fp = open(orig_fname, "a")
+        fp = open(orig_fname, "ab")
         fp.write(binascii.a2b_base64(data))
         fp.close()
 
diff --git a/elbepack/kvm.py b/elbepack/kvm.py
index 62f2ddeb..8ea0e6dd 100644
--- a/elbepack/kvm.py
+++ b/elbepack/kvm.py
@@ -6,7 +6,8 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import os
-import subprocess
+
+from elbepack.shellhelper import command_out
 
 kvm_exe_list = [
     '/usr/bin/kvm',
@@ -21,11 +22,8 @@ def find_kvm_exe():
     for fname in kvm_exe_list:
         if os.path.isfile(fname) and os.access(fname, os.X_OK):
             # determine kvm version
-            cmd = subprocess.Popen(
-                fname + ' --version',
-                shell=True,
-                stdout=subprocess.PIPE)
-            for line in cmd.stdout:
+            _, stdout = command_out(fname + ' --version')
+            for line in stdout.splitlines():
                 if "version" in line:
                     version = line.split()[3].split('(')[0].strip()
 
diff --git a/elbepack/log.py b/elbepack/log.py
index 43e04985..bffc6c1d 100644
--- a/elbepack/log.py
+++ b/elbepack/log.py
@@ -11,6 +11,7 @@ import select
 import threading
 from contextlib import contextmanager
 
+from io import TextIOWrapper, BytesIO
 
 root = logging.getLogger()
 root.setLevel(logging.DEBUG)
@@ -241,7 +242,7 @@ class AsyncLogging(object):
 
     def run(self):
         alive = True
-        rest = ""
+        rest = bytes()
         while alive:
             events = self.epoll.poll()
             for _, event in events:
@@ -256,6 +257,8 @@ class AsyncLogging(object):
             if not rest:
                 break
 
+        rest = TextIOWrapper(BytesIO(rest), encoding='utf-8', errors='replace').read()
+
         if self.lines:
             self.lines[-1] += rest
             self.block.info("\n".join(self.lines))
diff --git a/elbepack/shellhelper.py b/elbepack/shellhelper.py
index 0895c0b0..c73642c7 100644
--- a/elbepack/shellhelper.py
+++ b/elbepack/shellhelper.py
@@ -12,6 +12,7 @@ from subprocess import Popen, PIPE, STDOUT, call
 
 from elbepack.log import async_logging
 
+from io import TextIOWrapper, BytesIO
 
 log = logging.getLogger("log")
 soap = logging.getLogger("soap")
@@ -54,6 +55,8 @@ def command_out(cmd, stdin=None, output=PIPE, env_add=None):
                   stdout=output, stderr=STDOUT, stdin=PIPE, env=new_env)
         out, _ = p.communicate(input=stdin)
 
+    out = TextIOWrapper(BytesIO(out), encoding='utf-8', errors='replace').read()
+
     return p.returncode, out
 
 
@@ -81,6 +84,9 @@ def command_out_stderr(cmd, stdin=None, env_add=None):
                   stdout=PIPE, stderr=PIPE, stdin=PIPE, env=new_env)
         output, stderr = p.communicate(input=stdin)
 
+    output = TextIOWrapper(BytesIO(output), encoding='utf-8', errors='replace').read()
+    stderr = TextIOWrapper(BytesIO(stderr), encoding='utf-8', errors='replace').read()
+
     return p.returncode, output, stderr
 
 
diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
index c654e410..cb8dbd8f 100644
--- a/elbepack/soapclient.py
+++ b/elbepack/soapclient.py
@@ -87,7 +87,7 @@ class ElbeSoapClient(object):
         self.service.login(user, passwd)
 
     def download_file(self, builddir, filename, dst_fname):
-        fp = open(dst_fname, "w")
+        fp = open(dst_fname, "wb")
         part = 0
 
         # XXX the retry logic might get removed in the future, if the error
@@ -327,7 +327,7 @@ class SetXmlAction(ClientAction):
         part = 0
         with open(filename, "rb") as fp:
             while True:
-                xml_base64 = binascii.b2a_base64(fp.read(size))
+                xml_base64 = binascii.b2a_base64(fp.read(size)).decode('ascii')
                 # finish upload
                 if len(xml_base64) == 1:
                     part = client.service.upload_file(builddir,
-- 
2.20.1