[elbe-devel] [PATCH v3 15/52] Busy wait read from project log queue

[dion at linutronix.de]

From: Olivier Dion <dion at linutronix.de>

Instead of reading the log file of a project, the wait_busy action
will now pull the project' message queue (see QHandler in "log.py").

If there's no message, wait_busy sleep for 0.1 second before trying
again.  Using the queue and sleeping should reduce the CPU usage of
the user drastically.

The special string "ELBE-FINISH", which shoudn't in theory appear as a
single line logged message, act as a sentinel to terminate the
wait_busy loop.  Of course if the SOAP server could allow to return
back a boolean and string that would be awesome .. but this hack work
for now.

* NOTE

  If multiple wait_busy instances try to pull from the *same* project'
  message queue, then they will fight for messages and can result in
  no message for one of the instance even though the project is
  producing outputs.

Signed-off-by: Olivier Dion <dion at linutronix.de>
---
 elbepack/daemons/soap/esoap.py | 13 +++++--------
 elbepack/projectmanager.py     | 24 ++++--------------------
 elbepack/soapclient.py         | 31 +++++++------------------------
 3 files changed, 16 insertions(+), 52 deletions(-)

diff --git a/elbepack/daemons/soap/esoap.py b/elbepack/daemons/soap/esoap.py
index 09fb45fb..d71d069d 100644
--- a/elbepack/daemons/soap/esoap.py
+++ b/elbepack/daemons/soap/esoap.py
@@ -365,18 +365,15 @@ class ESoap (ServiceBase):
     def new_project(self, uid):
         return self.app.pm.new_project(uid)
 
-    @rpc(String, Integer, _returns=String)
+    @rpc(String, _returns=String)
     @authenticated_uid
     @soap_faults
-    def get_project_busy(self, uid, builddir, part):
+    def get_project_busy(self, uid, builddir):
         self.app.pm.open_project(uid, builddir)
-        ret, log = self.app.pm.current_project_is_busy(uid, part)
-        # return bool value to be compatible with elbe v1.0
-        if (part is None) and (log == "") and (not ret):
-            return ret
+        ret, msg = self.app.pm.current_project_is_busy(uid)
         if not ret:
-            return 'FINISH'
-        return log
+            return 'ELBE-FINISH'
+        return msg
 
     @rpc()
     @authenticated_uid
diff --git a/elbepack/projectmanager.py b/elbepack/projectmanager.py
index 49cc2945..0e241a66 100644
--- a/elbepack/projectmanager.py
+++ b/elbepack/projectmanager.py
@@ -25,6 +25,7 @@ from elbepack.asyncworker import (AsyncWorker, BuildJob, APTUpdateJob,
                                   BuildSDKJob, BuildCDROMsJob)
 
 from elbepack.elbexml import ValidationMode
+from elbepack.log import read_loggingQ
 
 
 class ProjectManagerError(Exception):
@@ -518,28 +519,11 @@ class ProjectManager(object):
             builddir = self._get_current_project(userid).builddir
             return self.db.has_changes(builddir)
 
-    def current_project_is_busy(self, userid, part):
+    def current_project_is_busy(self, userid):
         with self.lock:
             ep = self._get_current_project(userid)
-            count = 0
-
-            # function is called with part=None for elbe 1.0 clients
-            if part is None:
-                return self.db.is_busy(ep.builddir), ""
-
-            logline = None
-            with open(os.path.join(ep.builddir, 'log.txt'), 'r', 0) as lf:
-                for logline in lf:
-                    logline = logline.decode('utf-8','replace')
-                    if count == part:
-                        logline = unicode(part + 1) + u'###' + logline
-                        return self.db.is_busy(ep.builddir), logline
-                    count = count + 1
-            # don't crash if logfile doesn't exist
-            if not logline:
-                logline = u'None'
-            logline = unicode(part) + u'###' + logline
-            return self.db.is_busy(ep.builddir), logline
+            msg = read_loggingQ(ep.builddir)
+            return self.db.is_busy(ep.builddir), msg
 
     def _get_current_project(self, userid, allow_busy=True):
         # Must be called with self.lock held
diff --git a/elbepack/soapclient.py b/elbepack/soapclient.py
index 43b8f6e2..edcffe76 100644
--- a/elbepack/soapclient.py
+++ b/elbepack/soapclient.py
@@ -583,11 +583,10 @@ class WaitProjectBusyAction(ClientAction):
             sys.exit(20)
 
         builddir = args[0]
-        part = 1
 
         while True:
             try:
-                busy = client.service.get_project_busy(builddir, part)
+                msg = client.service.get_project_busy(builddir)
             # TODO the root cause of this problem is unclear. To enable a
             # get more information print the exception and retry to see if
             # the connection problem is just a temporary problem. This
@@ -599,30 +598,14 @@ class WaitProjectBusyAction(ClientAction):
                       file=sys.stderr)
                 continue
 
-            if busy == 'FINISH':
+            if not msg:
+                time.sleep(0.1)
+                continue
+
+            if msg == 'ELBE-FINISH':
                 break
-            else:
-                # for some reasons lines containing e.g. ^H result in a None
-                # object here. let's just skip those strange lines for the
-                # moment
-                if busy:
-                    log = busy.split('###')
-
-                    if part != int(log[0]):
-                        part = int(log[0])
-
-                        localtime = time.asctime(time.localtime(time.time()))
-                        try:
-                            print("%s -- %s" % (localtime,
-                                                log[1].replace('\n','')))
-                        except IndexError:
-                            print("IndexError - part: %d (skipped)" % part)
-                    else:
-                        time.sleep(1)
-                else:
-                    print("strange part: %d (skipped)" % part)
-                    part = part + 1
 
+            print(msg)
 
 ClientAction.register(WaitProjectBusyAction)
 
-- 
2.11.0