[elbe-devel] [PATCH v4 1/1] Python3: fix encoding for both py2 and py3

[Christian Teklenborg]

Python3 has unicode strings as default whereas Python2 has binary strings.
To avoid TypeErrors change the encoding at the necessary places.
At some points we need only unicode strings but a string in Python3 doesn't have
a decode() function and we need this decode() function to get a unicode string
from a Python2 binary string. Python2 strings have an encode function, thus use
.encode().decode() to make sure we end up with a unicode string in both cases.

Again, this gets rid of the last unicode() calls which are not available on
Python3.

Signed-off-by: Christian Teklenborg <chris at linutronix.de>
---
 elbepack/daemons/soap/esoap.py |  2 +-
 elbepack/efilesystem.py        | 13 ++++++++-----
 elbepack/elbexml.py            |  4 ++--
 elbepack/licencexml.py         |  9 +++++++--
 elbepack/rfs.py                |  4 ++--
 elbepack/soapclient.py         |  8 ++++----
 6 files changed, 24 insertions(+), 16 deletions(-)

diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py
index 7d76d6a3..ddc09018 100644
--- a/elbepack/daemons/soap/esoap.py
+++ b/elbepack/daemons/soap/esoap.py
@@ -178,7 +178,7 @@ class ESoap (ServiceBase):
         if pos >= file_stat.st_size:
             return "EndOfFile"
 
-        with open(file_name) as fp:
+        with open(file_name, 'rb') as fp:
             if not fp:
                 return "FileNotFound"
             try:
diff --git a/elbepack/efilesystem.py b/elbepack/efilesystem.py
index 5bc20afd..0c3b03e0 100644
--- a/elbepack/efilesystem.py
+++ b/elbepack/efilesystem.py
@@ -147,18 +147,21 @@ class ElbeFilesystem(Filesystem):
             copyright_file = os.path.join('/usr/share/doc', pkg, 'copyright')
             copyright_fname = self.fname(copyright_file)
             try:
-                with io.open(copyright_fname, "rb") as lic:
+                with io.open(copyright_fname, "r",
+                             encoding='utf-8', errors='replace') as lic:
                     lic_text = lic.read()
             except IOError as e:
                 logging.exception("Error while processing license file %s",
                                   copyright_fname)
-                lic_text = "Error while processing license file %s: '%s'" % (
+                lic_text = u"Error while processing license file %s: '%s'" % (
                     copyright_file, e.strerror)
-
-            lic_text = unicode(lic_text, encoding='utf-8', errors='replace')
+            # in Python2 'pkg' is a binary string whereas in Python3 it is a
+            # unicode string. So make sure that pkg ends up as a unicode string
+            # in both Python2 and Python3.
+            pkg = pkg.encode(encoding='utf-8').decode(encoding='utf-8')
 
             if f is not None:
-                f.write(unicode(pkg))
+                f.write(pkg)
                 f.write(u":\n======================================"
                         "==========================================")
                 f.write(u"\n")
diff --git a/elbepack/elbexml.py b/elbepack/elbexml.py
index 9029062a..fa815839 100644
--- a/elbepack/elbexml.py
+++ b/elbepack/elbexml.py
@@ -193,13 +193,13 @@ class ElbeXML(object):
         ret = False
         if "srcstr" in r:
             for line in fp:
-                needle = bytes(r["srcstr"])
+                needle = r["srcstr"].encode(encoding='utf-8')
                 if line.find(needle) != -1:
                     ret = True
                     break
         elif "binstr" in r:
             for line in fp:
-                needle = bytes(r["binstr"])
+                needle = r["binstr"].encode(encoding='utf-8')
                 if line.find(needle) != -1:
                     ret = True
                     break
diff --git a/elbepack/licencexml.py b/elbepack/licencexml.py
index 6324fe4f..f5f4ee5e 100644
--- a/elbepack/licencexml.py
+++ b/elbepack/licencexml.py
@@ -59,8 +59,13 @@ class copyright_xml (object):
         xmlpkg.et.attrib['name'] = pkg_name
         txtnode = xmlpkg.append('text')
         txtnode.et.text = copyright_text
-
-        bytesio = io.StringIO(unicode(txtnode.et.text))
+        # in Python2 'txtnode.et.text' is a binary string whereas in Python3 it
+        # is a unicode string. So make sure that 'txtnode.et.text' ends up as a
+        # unicode string in both Python2 and Python3.
+        bytesio = io.StringIO(txtnode.et.text.encode(encoding='utf-8',
+                                                     errors='replace')
+                                             .decode(encoding='utf-8',
+                                                     errors='replace'))
         try:
             c = Copyright(bytesio)
             files = []
diff --git a/elbepack/rfs.py b/elbepack/rfs.py
index c65d9831..6967d289 100644
--- a/elbepack/rfs.py
+++ b/elbepack/rfs.py
@@ -186,7 +186,7 @@ class BuildEnv (object):
         if arch == "default":
             arch = self.xml.text("project/buildimage/arch", key="arch")
 
-        host_arch = get_command_out("dpkg --print-architecture").strip()
+        host_arch = get_command_out("dpkg --print-architecture").strip().decode()
 
         includepkgs = None
         strapcmd  = 'debootstrap '
@@ -317,7 +317,7 @@ class BuildEnv (object):
 
 
     def seed_etc(self):
-        passwd = self.xml.text("target/passwd")
+        passwd = self.xml.text("target/passwd").encode(encoding='utf-8')
         stdin = "%s\n%s\n" % (passwd, passwd)
         chroot(self.rfs.path, "passwd", stdin=stdin)
 
diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
index 589bdefd..586fa513 100644
--- a/elbepack/soapclient.py
+++ b/elbepack/soapclient.py
@@ -454,15 +454,15 @@ class GetFileAction(ClientAction):
                 file=sys.stderr)
             sys.exit(20)
 
-        builddir = args[0]
-        filename = args[1]
-        dst_fname = filename
+        builddir = args[0].encode()
+        filename = args[1].encode()
+        dst_fname = filename.encode()
 
         if opt.output:
             fs = Filesystem('/')
             dst = os.path.abspath(opt.output)
             fs.mkdir_p(dst)
-            dst_fname = str(os.path.join(dst, filename))
+            dst_fname = str(os.path.join(dst, filename)).encode()
 
         client.download_file(builddir, filename, dst_fname)
         print("%s saved" % dst_fname)
-- 
2.20.1