Return cached worker info immediately, update in the background

Workers already update their state when it changes, and this state
propagates through OBS proxies, so we can safely return it immediately
when requested by the warden, and schedule an update in the background.

This should mitigate timeouts when the warden prods OBS proxy server
prods OBS proxy client prods worker, and while the reply climbs back
through the interpipes, warden times out and kills the job.

Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>

obs_proxy/server.py

@@ -423,19 +423,13 @@ async def worker_state(worker_id: str = None):
         return workers[worker_id].state
 
 
-@app.route('/worker')
-@ext_resources.route('/worker/<path:worker_id>/info')
-@require_auth
-@per_worker
-async def worker_info(worker_id: str = None):
-    if worker_id not in ws_connections:
-        return 'not found', 404
-    params = dict(request.args)
+async def update_worker_info(params):
+    worker_id = params['worker_id']
     orig_job_id = params.get('jobid', None)
     params['jobid'] = map_jobid_to_internal(worker_id, orig_job_id)
 
-    params['worker_id'] = worker_id
     job_trace("server", orig_job_id, worker_id, f"=> info {params['jobid']}")
+    logger.debug("=> info", params=params)
     resp = await ws_connections[worker_id].call(
         JSONRPC20Request(
             method='worker_info',
@@ -444,14 +438,33 @@ async def worker_info(worker_id: str = None):
     )
     job_trace("server", orig_job_id, worker_id, f"<= {resp.data}")
     if resp.error:
-        return resp.error, 500
+        logger.error("error in worker update", error=resp.error)
+        return
     w = Worker.fromdict(resp.result['worker'])
+    logger.debug("updating worker", info=w)
     workers[worker_id].update(w)
+
+
+@app.route('/worker')
+@ext_resources.route('/worker/<path:worker_id>/info')
+@require_auth
+@per_worker
+async def worker_info(worker_id: str = None):
+    if worker_id not in ws_connections:
+        return 'not found', 404
+    params = dict(request.args)
+
+    if worker_id not in workers:
+        logger.warning("worker in ws_connections but not in workers?!")
+        return 'not found', 404
+
+    params['worker_id'] = worker_id
+
+    logger.debug("updating worker in background", params=params)
+    app.add_background_task(update_worker_info, params)
+
     w = workers[worker_id]
-    if resp.result['code'] == 200:
-        return f"{w.external().asxml(pure=True, meta=False)}", {'content-type': 'text/xml'}
-    else:
-        return resp.result.get('content', ''), resp.result['code']
+    return w.external().asxml(pure=True, meta=False), {'content-type': 'text/xml'}
 
 
 @app.route('/worker/<path:worker_id>/status')