[elbe-devel] [PATCH v3 15/52] Busy wait read from project log queue
Torben Hohn
torben.hohn at linutronix.de
Fri Jul 5 10:54:33 CEST 2019
On Thu, Jun 27, 2019 at 02:44:29PM +0200, dion at linutronix.de wrote:
> From: Olivier Dion <dion at linutronix.de>
>
> Instead of reading the log file of a project, the wait_busy action
> will now pull the project' message queue (see QHandler in "log.py").
>
> If there's no message, wait_busy sleep for 0.1 second before trying
> again. Using the queue and sleeping should reduce the CPU usage of
> the user drastically.
>
> The special string "ELBE-FINISH", which shoudn't in theory appear as a
> single line logged message, act as a sentinel to terminate the
> wait_busy loop. Of course if the SOAP server could allow to return
> back a boolean and string that would be awesome .. but this hack work
> for now.
>
> * NOTE
>
> If multiple wait_busy instances try to pull from the *same* project'
> message queue, then they will fight for messages and can result in
> no message for one of the instance even though the project is
> producing outputs.
>
> Signed-off-by: Olivier Dion <dion at linutronix.de>
Reviewed-by: Torben Hohn <torben.hohn at linutronix.de>
> ---
> elbepack/daemons/soap/esoap.py | 13 +++++--------
> elbepack/projectmanager.py | 24 ++++--------------------
> elbepack/soapclient.py | 31 +++++++------------------------
> 3 files changed, 16 insertions(+), 52 deletions(-)
>
> diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py
> index 09fb45fb..d71d069d 100644
> --- a/elbepack/daemons/soap/esoap.py
> +++ b/elbepack/daemons/soap/esoap.py
> @@ -365,18 +365,15 @@ class ESoap (ServiceBase):
> def new_project(self, uid):
> return self.app.pm.new_project(uid)
>
> - @rpc(String, Integer, _returns=String)
> + @rpc(String, _returns=String)
> @authenticated_uid
> @soap_faults
> - def get_project_busy(self, uid, builddir, part):
> + def get_project_busy(self, uid, builddir):
> self.app.pm.open_project(uid, builddir)
> - ret, log = self.app.pm.current_project_is_busy(uid, part)
> - # return bool value to be compatible with elbe v1.0
> - if (part is None) and (log == "") and (not ret):
> - return ret
> + ret, msg = self.app.pm.current_project_is_busy(uid)
> if not ret:
> - return 'FINISH'
> - return log
> + return 'ELBE-FINISH'
> + return msg
>
> @rpc()
> @authenticated_uid
> diff --git a/elbepack/projectmanager.py b/elbepack/projectmanager.py
> index 49cc2945..0e241a66 100644
> --- a/elbepack/projectmanager.py
> +++ b/elbepack/projectmanager.py
> @@ -25,6 +25,7 @@ from elbepack.asyncworker import (AsyncWorker, BuildJob, APTUpdateJob,
> BuildSDKJob, BuildCDROMsJob)
>
> from elbepack.elbexml import ValidationMode
> +from elbepack.log import read_loggingQ
>
>
> class ProjectManagerError(Exception):
> @@ -518,28 +519,11 @@ class ProjectManager(object):
> builddir = self._get_current_project(userid).builddir
> return self.db.has_changes(builddir)
>
> - def current_project_is_busy(self, userid, part):
> + def current_project_is_busy(self, userid):
> with self.lock:
> ep = self._get_current_project(userid)
> - count = 0
> -
> - # function is called with part=None for elbe 1.0 clients
> - if part is None:
> - return self.db.is_busy(ep.builddir), ""
> -
> - logline = None
> - with open(os.path.join(ep.builddir, 'log.txt'), 'r', 0) as lf:
> - for logline in lf:
> - logline = logline.decode('utf-8','replace')
> - if count == part:
> - logline = unicode(part + 1) + u'###' + logline
> - return self.db.is_busy(ep.builddir), logline
> - count = count + 1
> - # don't crash if logfile doesn't exist
> - if not logline:
> - logline = u'None'
> - logline = unicode(part) + u'###' + logline
> - return self.db.is_busy(ep.builddir), logline
> + msg = read_loggingQ(ep.builddir)
> + return self.db.is_busy(ep.builddir), msg
>
> def _get_current_project(self, userid, allow_busy=True):
> # Must be called with self.lock held
> diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
> index 43b8f6e2..edcffe76 100644
> --- a/elbepack/soapclient.py
> +++ b/elbepack/soapclient.py
> @@ -583,11 +583,10 @@ class WaitProjectBusyAction(ClientAction):
> sys.exit(20)
>
> builddir = args[0]
> - part = 1
>
> while True:
> try:
> - busy = client.service.get_project_busy(builddir, part)
> + msg = client.service.get_project_busy(builddir)
> # TODO the root cause of this problem is unclear. To enable a
> # get more information print the exception and retry to see if
> # the connection problem is just a temporary problem. This
> @@ -599,30 +598,14 @@ class WaitProjectBusyAction(ClientAction):
> file=sys.stderr)
> continue
>
> - if busy == 'FINISH':
> + if not msg:
> + time.sleep(0.1)
> + continue
> +
> + if msg == 'ELBE-FINISH':
> break
> - else:
> - # for some reasons lines containing e.g. ^H result in a None
> - # object here. let's just skip those strange lines for the
> - # moment
> - if busy:
> - log = busy.split('###')
> -
> - if part != int(log[0]):
> - part = int(log[0])
> -
> - localtime = time.asctime(time.localtime(time.time()))
> - try:
> - print("%s -- %s" % (localtime,
> - log[1].replace('\n','')))
> - except IndexError:
> - print("IndexError - part: %d (skipped)" % part)
> - else:
> - time.sleep(1)
> - else:
> - print("strange part: %d (skipped)" % part)
> - part = part + 1
>
> + print(msg)
>
> ClientAction.register(WaitProjectBusyAction)
>
> --
> 2.11.0
>
>
> _______________________________________________
> elbe-devel mailing list
> elbe-devel at linutronix.de
> https://lists.linutronix.de/mailman/listinfo/elbe-devel
--
Torben Hohn
Linutronix GmbH | Bahnhofstrasse 3 | D-88690 Uhldingen-Mühlhofen
Phone: +49 7556 25 999 18; Fax.: +49 7556 25 999 99
Hinweise zum Datenschutz finden Sie hier (Informations on data privacy
can be found here): https://linutronix.de/kontakt/Datenschutz.php
Linutronix GmbH | Firmensitz (Registered Office): Uhldingen-Mühlhofen |
Registergericht (Registration Court): Amtsgericht Freiburg i.Br., HRB700
806 | Geschäftsführer (Managing Directors): Heinz Egger, Thomas Gleixner
More information about the elbe-devel
mailing list