[elbe-devel] [PATCH v4 1/1] Python3: fix encoding for both py2 and py3

[Bastian Germann]

Am 14.02.20 um 15:50 schrieb Christian Teklenborg:
> Python3 has unicode strings as default whereas Python2 has binary strings.
> To avoid TypeErrors change the encoding at the necessary places.
> At some points we need only unicode strings but a string in Python3 doesn't have
> a decode() function and we need this decode() function to get a unicode string
> from a Python2 binary string. Python2 strings have an encode function, thus use
> .encode().decode() to make sure we end up with a unicode string in both cases.
> 
> Again, this gets rid of the last unicode() calls which are not available on
> Python3.
> 
> Signed-off-by: Christian Teklenborg <chris at linutronix.de>

Reviewed-by: Bastian Germann <bage at linutronix.de>

> ---
>  elbepack/daemons/soap/esoap.py |  2 +-
>  elbepack/efilesystem.py        | 13 ++++++++-----
>  elbepack/elbexml.py            |  4 ++--
>  elbepack/licencexml.py         |  9 +++++++--
>  elbepack/rfs.py                |  4 ++--
>  elbepack/soapclient.py         |  8 ++++----
>  6 files changed, 24 insertions(+), 16 deletions(-)
> 
> diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py
> index 7d76d6a3..ddc09018 100644
> --- a/elbepack/daemons/soap/esoap.py
> +++ b/elbepack/daemons/soap/esoap.py
> @@ -178,7 +178,7 @@ class ESoap (ServiceBase):
>          if pos >= file_stat.st_size:
>              return "EndOfFile"
>  
> -        with open(file_name) as fp:
> +        with open(file_name, 'rb') as fp:
>              if not fp:
>                  return "FileNotFound"
>              try:
> diff --git a/elbepack/efilesystem.py b/elbepack/efilesystem.py
> index 5bc20afd..0c3b03e0 100644
> --- a/elbepack/efilesystem.py
> +++ b/elbepack/efilesystem.py
> @@ -147,18 +147,21 @@ class ElbeFilesystem(Filesystem):
>              copyright_file = os.path.join('/usr/share/doc', pkg, 'copyright')
>              copyright_fname = self.fname(copyright_file)
>              try:
> -                with io.open(copyright_fname, "rb") as lic:
> +                with io.open(copyright_fname, "r",
> +                             encoding='utf-8', errors='replace') as lic:
>                      lic_text = lic.read()
>              except IOError as e:
>                  logging.exception("Error while processing license file %s",
>                                    copyright_fname)
> -                lic_text = "Error while processing license file %s: '%s'" % (
> +                lic_text = u"Error while processing license file %s: '%s'" % (
>                      copyright_file, e.strerror)
> -
> -            lic_text = unicode(lic_text, encoding='utf-8', errors='replace')
> +            # in Python2 'pkg' is a binary string whereas in Python3 it is a
> +            # unicode string. So make sure that pkg ends up as a unicode string
> +            # in both Python2 and Python3.
> +            pkg = pkg.encode(encoding='utf-8').decode(encoding='utf-8')
>  
>              if f is not None:
> -                f.write(unicode(pkg))
> +                f.write(pkg)
>                  f.write(u":\n======================================"
>                          "==========================================")
>                  f.write(u"\n")
> diff --git a/elbepack/elbexml.py b/elbepack/elbexml.py
> index 9029062a..fa815839 100644
> --- a/elbepack/elbexml.py
> +++ b/elbepack/elbexml.py
> @@ -193,13 +193,13 @@ class ElbeXML(object):
>          ret = False
>          if "srcstr" in r:
>              for line in fp:
> -                needle = bytes(r["srcstr"])
> +                needle = r["srcstr"].encode(encoding='utf-8')
>                  if line.find(needle) != -1:
>                      ret = True
>                      break
>          elif "binstr" in r:
>              for line in fp:
> -                needle = bytes(r["binstr"])
> +                needle = r["binstr"].encode(encoding='utf-8')
>                  if line.find(needle) != -1:
>                      ret = True
>                      break
> diff --git a/elbepack/licencexml.py b/elbepack/licencexml.py
> index 6324fe4f..f5f4ee5e 100644
> --- a/elbepack/licencexml.py
> +++ b/elbepack/licencexml.py
> @@ -59,8 +59,13 @@ class copyright_xml (object):
>          xmlpkg.et.attrib['name'] = pkg_name
>          txtnode = xmlpkg.append('text')
>          txtnode.et.text = copyright_text
> -
> -        bytesio = io.StringIO(unicode(txtnode.et.text))
> +        # in Python2 'txtnode.et.text' is a binary string whereas in Python3 it
> +        # is a unicode string. So make sure that 'txtnode.et.text' ends up as a
> +        # unicode string in both Python2 and Python3.
> +        bytesio = io.StringIO(txtnode.et.text.encode(encoding='utf-8',
> +                                                     errors='replace')
> +                                             .decode(encoding='utf-8',
> +                                                     errors='replace'))
>          try:
>              c = Copyright(bytesio)
>              files = []
> diff --git a/elbepack/rfs.py b/elbepack/rfs.py
> index c65d9831..6967d289 100644
> --- a/elbepack/rfs.py
> +++ b/elbepack/rfs.py
> @@ -186,7 +186,7 @@ class BuildEnv (object):
>          if arch == "default":
>              arch = self.xml.text("project/buildimage/arch", key="arch")
>  
> -        host_arch = get_command_out("dpkg --print-architecture").strip()
> +        host_arch = get_command_out("dpkg --print-architecture").strip().decode()
>  
>          includepkgs = None
>          strapcmd  = 'debootstrap '
> @@ -317,7 +317,7 @@ class BuildEnv (object):
>  
>  
>      def seed_etc(self):
> -        passwd = self.xml.text("target/passwd")
> +        passwd = self.xml.text("target/passwd").encode(encoding='utf-8')
>          stdin = "%s\n%s\n" % (passwd, passwd)
>          chroot(self.rfs.path, "passwd", stdin=stdin)
>  
> diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
> index 589bdefd..586fa513 100644
> --- a/elbepack/soapclient.py
> +++ b/elbepack/soapclient.py
> @@ -454,15 +454,15 @@ class GetFileAction(ClientAction):
>                  file=sys.stderr)
>              sys.exit(20)
>  
> -        builddir = args[0]
> -        filename = args[1]
> -        dst_fname = filename
> +        builddir = args[0].encode()
> +        filename = args[1].encode()
> +        dst_fname = filename.encode()
>  
>          if opt.output:
>              fs = Filesystem('/')
>              dst = os.path.abspath(opt.output)
>              fs.mkdir_p(dst)
> -            dst_fname = str(os.path.join(dst, filename))
> +            dst_fname = str(os.path.join(dst, filename)).encode()
>  
>          client.download_file(builddir, filename, dst_fname)
>          print("%s saved" % dst_fname)
>