[elbe-devel] [PATCH 4/5] Python3: convert bytes to unicode
Torben Hohn
torben.hohn at linutronix.de
Fri Dec 6 17:22:37 CET 2019
On Fri, Dec 06, 2019 at 02:08:48PM +0100, Christian Teklenborg wrote:
> changed the byte strings to unicode strings and vice-versa to make it Python2
> and Python3 compatible
errrm, vice-versa ?
python3 changes the type of a lot of vriables, which have been "str" in
python2 to "bytes" these have to be converted to "str" which cintains
unicode characters. (like the type "unicode" in py2)
Please make your explanation more exact.
>
> Signed-off-by: Christian Teklenborg <chris at linutronix.de>
> ---
> elbepack/commands/init.py | 6 +++---
> elbepack/daemons/soap/esoap.py | 10 +++++-----
> elbepack/initvmaction.py | 2 +-
> elbepack/kvm.py | 1 +
> elbepack/log.py | 2 +-
> elbepack/shellhelper.py | 2 +-
> elbepack/soapclient.py | 4 ++--
> 7 files changed, 14 insertions(+), 13 deletions(-)
>
> diff --git a/elbepack/commands/init.py b/elbepack/commands/init.py
> index 46cd8dcd..90429943 100644
> --- a/elbepack/commands/init.py
> +++ b/elbepack/commands/init.py
> @@ -150,9 +150,9 @@ def run_command(argv):
>
> # this is a workaround for
> # http://lists.linutronix.de/pipermail/elbe-devel/2017-July/000541.html
> - _, virt = command_out('test -x /usr/bin/systemd-detect-virt && '
> - '/usr/bin/systemd-detect-virt')
> - _, dist = command_out('lsb_release -cs')
> + virt = command_out('test -x /usr/bin/systemd-detect-virt && '
> + '/usr/bin/systemd-detect-virt')[1].decode()
> + dist = command_out('lsb_release -cs')[1].decode()
adding [1].decode() really makes a line a lot harder to read.
it already does 3 things at once. if you add 2 more things, that becomes
5.
anyways. we really want to fix command_out to return decoded() strings.
i dont see any use of command_out() where we want it to return "bytes"
ALSO NOTE:
----------
decode() can throw a UnicodeDecodeError when the input is no valid
UTF-8. We fell over this pretty often in the past.
Either you catch that Exception and do something sensible.
Or use decode(errors="replace") <- easier
>
> if 'vmware' in virt and 'stretch' in dist:
> machine_type = 'pc-i440fx-2.6'
> diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py
> index 468df8a5..450abf0e 100644
> --- a/elbepack/daemons/soap/esoap.py
> +++ b/elbepack/daemons/soap/esoap.py
> @@ -160,7 +160,7 @@ class ESoap (ServiceBase):
> return -2
>
> with open(fn, 'a') as fp:
> - fp.write(binascii.a2b_base64(blob))
> + fp.write(binascii.a2b_base64(blob).decode())
i dont understand, why this should be converted to string ?
fp.write() should be wanting bytes()
>
> return part + 1
>
> @@ -265,7 +265,7 @@ class ESoap (ServiceBase):
>
> # Now append data to cdrom_file
> fp = open(cdrom_fname, "a")
> - fp.write(binascii.a2b_base64(data))
> + fp.write(binascii.a2b_base64(data).decode())
> fp.close()
i dont understand, why this should be converted to string ?
fp.write() should be wanting bytes()
>
> @rpc(String)
> @@ -298,7 +298,7 @@ class ESoap (ServiceBase):
>
> # Now write empty File
> fp = open(pdebuild_fname, "a")
> - fp.write(binascii.a2b_base64(data))
> + fp.write(binascii.a2b_base64(data).decode())
> fp.close()
i dont understand, why this should be converted to string ?
fp.write() should be wanting bytes()
>
> @rpc(String, Integer, String)
> @@ -332,7 +332,7 @@ class ESoap (ServiceBase):
>
> # Now append to File
> fp = open(orig_fname, "a")
> - fp.write(binascii.a2b_base64(data))
> + fp.write(binascii.a2b_base64(data).decode())
> fp.close()
i dont understand, why this should be converted to string ?
fp.write() should be wanting bytes()
>
> @rpc(String)
> @@ -362,7 +362,7 @@ class ESoap (ServiceBase):
> @soap_faults
> def create_project(self, uid, xml, url_validation):
> with NamedTemporaryFile() as fp:
> - fp.write(binascii.a2b_base64(xml))
> + fp.write(binascii.a2b_base64(xml).decode())
i dont understand, why this should be converted to string ?
fp.write() should be wanting bytes()
> fp.flush()
> prjid = self.app.pm.create_project(
> uid, fp.name, url_validation=url_validation)
> diff --git a/elbepack/initvmaction.py b/elbepack/initvmaction.py
> index f8572402..75fa81a6 100644
> --- a/elbepack/initvmaction.py
> +++ b/elbepack/initvmaction.py
> @@ -234,7 +234,7 @@ def submit_and_dl_result(xmlfile, cdrom, opt):
> print("Giving up", file=sys.stderr)
> sys.exit(20)
>
> - prjdir = prjdir.strip()
> + prjdir = prjdir.strip().decode()
dont call strip() before decode()
anyways... this would be fixed by a fixed command_out_stderr()
>
> cmd = '%s control set_xml %s %s' % (elbe_exe, prjdir, xmlfile)
> ret, _, err = command_out_stderr(cmd)
> diff --git a/elbepack/kvm.py b/elbepack/kvm.py
> index 62f2ddeb..58a17530 100644
> --- a/elbepack/kvm.py
> +++ b/elbepack/kvm.py
> @@ -24,6 +24,7 @@ def find_kvm_exe():
> cmd = subprocess.Popen(
> fname + ' --version',
> shell=True,
> + universal_newlines=True,
> stdout=subprocess.PIPE)
> for line in cmd.stdout:
> if "version" in line:
this could also be replaced by command_out()
> diff --git a/elbepack/log.py b/elbepack/log.py
> index 43e04985..67bac91f 100644
> --- a/elbepack/log.py
> +++ b/elbepack/log.py
> @@ -241,7 +241,7 @@ class AsyncLogging(object):
>
> def run(self):
> alive = True
> - rest = ""
> + rest = b""
please use bytes()
also look around carefuly here: There is no decode(). But we want
the self.lines() to contain strings. so we should call decode()
somewhere.
Maybe we should wrap the file object with TextIOWrapper()
> while alive:
> events = self.epoll.poll()
> for _, event in events:
> diff --git a/elbepack/shellhelper.py b/elbepack/shellhelper.py
> index 0895c0b0..955abb0b 100644
> --- a/elbepack/shellhelper.py
> +++ b/elbepack/shellhelper.py
> @@ -106,7 +106,7 @@ def do(cmd, allow_fail=False, stdin=None, env_add=None):
> if stdin is None:
> p = Popen(cmd, shell=True, stdout=w, stderr=STDOUT, env=new_env)
> else:
> - p = Popen(cmd, shell=True, stdin=PIPE, stdout=w, stderr=STDOUT, env=new_env)
> + p = Popen(cmd, shell=True, stdin=PIPE, stdout=w, stderr=STDOUT, env=new_env, universal_newlines=True)
I am not a fan of "universal_newlines=" it does not allow us to specify
errors="replace", that is only possible since python3.6 ... i still want
this to work on stretch though. and that has python-3.5.3
In shellhelper, we can use TextIOWrapper()
>
> async_logging(r, w, soap, log)
> p.communicate(input=stdin)
> diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
> index 62324cd0..c64926ef 100644
> --- a/elbepack/soapclient.py
> +++ b/elbepack/soapclient.py
> @@ -328,7 +328,7 @@ class SetXmlAction(ClientAction):
> part = 0
> with open(filename, "rb") as fp:
> while True:
> - xml_base64 = binascii.b2a_base64(fp.read(size))
> + xml_base64 = binascii.b2a_base64(fp.read(size)).decode()
> # finish upload
> if len(xml_base64) == 1:
> part = client.service.upload_file(builddir,
> @@ -519,7 +519,7 @@ class DumpFileAction(ClientAction):
> if ret == "EndOfFile":
> return
>
> - sys.stdout.write(binascii.a2b_base64(ret))
> + sys.stdout.write(binascii.a2b_base64(ret).decode())
> part = part + 1
>
>
> --
> 2.20.1
>
>
> _______________________________________________
> elbe-devel mailing list
> elbe-devel at linutronix.de
> https://lists.linutronix.de/mailman/listinfo/elbe-devel
--
Torben Hohn
Linutronix GmbH | Bahnhofstrasse 3 | D-88690 Uhldingen-Mühlhofen
Phone: +49 7556 25 999 18; Fax.: +49 7556 25 999 99
Hinweise zum Datenschutz finden Sie hier (Informations on data privacy
can be found here): https://linutronix.de/kontakt/Datenschutz.php
Linutronix GmbH | Firmensitz (Registered Office): Uhldingen-Mühlhofen |
Registergericht (Registration Court): Amtsgericht Freiburg i.Br., HRB700
806 | Geschäftsführer (Managing Directors): Heinz Egger, Thomas Gleixner
More information about the elbe-devel
mailing list