Update the test cases table automatically

Update the test cases table from the apertis-test-cases repository
automatically.

This commit rewrites the code to create a TestCasesTable object that
can be used to create and update the table conveniently from other modules.

The commit also creates a background process (using the apscheduler) in
order to pull updates from the apertis-test-cases repo (every 15
minutes) and update the table automatically.

The commit also apply the necessary changes in the Dockerfile to install
the new git and apscheduler packages, and in the code to generate the
report page.
Signed-off-by: Luis Araujo <luis.araujo@collabora.co.uk>

.gitlab-ci.yml

@@ -3,7 +3,7 @@ image: debian:buster
 before_script:
   - export DEBIAN_FRONTEND=noninteractive
   - apt-get update
-  - apt-get install --no-install-recommends -y python3-setuptools python3-wheel python3-pip python3-yaml python3-phabricator python3-yattag ca-certificates python3-flask python3-flask-migrate python3-flask-script python3-flask-sqlalchemy python3-psycopg2 python3-oauthlib python3-blinker python3-requests-oauthlib python3-urlobject
+  - apt-get install --no-install-recommends -y python3-setuptools python3-wheel python3-pip python3-yaml python3-phabricator python3-yattag ca-certificates python3-flask python3-flask-migrate python3-flask-script python3-flask-sqlalchemy python3-psycopg2 python3-oauthlib python3-blinker python3-requests-oauthlib python3-urlobject python3-apscheduler
   - apt-get install --no-install-recommends -y python3-lxml python3-pytest python3-testing.postgresql
   - pip3 install Flask-Dance
   - adduser --no-create-home --disabled-password --gecos "" testuser

Dockerfile

@@ -5,7 +5,9 @@ ARG DEBIAN_FRONTEND=noninteractive
 RUN apt update && \
     apt install --no-install-recommends -y \
         ca-certificates \
+        git \
         python3-all \
+        python3-apscheduler \
         python3-blinker \
         python3-flask \
         python3-flask-migrate \
@@ -44,6 +46,9 @@ FROM base AS runtime
 
 LABEL description="Apertis QA test results collection and reporting application - runtime environment"
 
+RUN git clone https://gitlab.apertis.org/tests/apertis-test-cases.git \
+              /var/lib/apertis-test-cases
+
 WORKDIR /app
 
 ADD . /app

data/generate_tc_table.py

-#!/usr/bin/env python3
-#
-# Tool to generate the test case -> images table.
-#
-# This script generates a list structure to properly map the test cases to the list
-# of their supported image types:
-#
-# [(test-case-name0, [ deployment-type0, deployment-type1 ... ],
-#                    [ image-type0, image-type1, image-type2 ... ]),
-#  (test-case-name1, [ deployment-type0, deployment-type1 ... ],
-#                    [ image-type0, image-type1, image-type2 ... ]),
-#  ...
-# ]
-#
-# The script should be executed in the following way passing the test-cases/
-# directory:
-#
-#    $ ./generate_tc_table test-cases/ > tc_table.py
-#
-# Then the tc_table.py can be imported from the module generating the report page
-# (reportpage.py).
-#
-# Copyright (C) 2019 Collabora Ltd
-# Luis Araujo <luis.araujo@collabora.co.uk>
-
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  US
-###################################################################################
-
-
-import os
-import sys
-import yaml
-
-
-list_table = []
-
-priority_color = {
-    'low'      : 'secondary',
-    'medium'   : 'info',
-    'high'     : 'warning',
-    'critical' : 'danger'
-}
-
-def update_table(tc_file):
-    try:
-        with open(tc_file) as testcase:
-            tc_data = yaml.safe_load(testcase)
-    except yaml.scanner.ScannerError as e:
-        print("yaml format error:", e)
-        exit(1)
-
-    # Only automated tests are currently support, so skip manual tests.
-    # TODO: Manual tests should be added later.
-    if tc_data['metadata']['exec-type'] == 'manual':
-        return
-
-    tc_name = tc_data['metadata']['name']
-    image_type_platforms = []
-    for image_type , platforms in tc_data['metadata']['image-types'].items():
-        for platform in platforms:
-            # Apply this change to public jobs to keep consistency with the DB:
-            #    'minimal-armhf' => 'minimal-armhf-public'
-            v = platform.split('-')[-1]
-            if v != 'internal':
-                platform += '-public'
-            image_type_platforms.append(image_type + '-' + platform)
-
-    image_deployments = [ e.lower() for e in
-                          tc_data['metadata']['image-deployment'] ]
-    list_table.append((tc_name, priority_color[tc_data['metadata']['priority']],
-                       image_deployments, image_type_platforms))
-
-if __name__ == '__main__':
-    for root, _, files in os.walk(sys.argv[1]):
-        for f in files:
-            tc_file = os.path.join(root, f)
-            update_table(tc_file)
-
-    print("tc_table =", sorted(list_table))

data/testcases_table.py

+#!/usr/bin/env python3
+#
+# This module contains the TestCasesTable object which represents the main
+# test cases table with an internal structure of the form:
+#
+# [(test-case-name0, priority, [ deployment-type0, deployment-type1 ... ],
+#                              [ image-type0, image-type1, image-type2 ... ]),
+#  (test-case-name1, priority, [ deployment-type0, deployment-type1 ... ],
+#                              [ image-type0, image-type1, image-type2 ... ]),
+#  ...
+# ]
+#
+# This structure is referenced by the `self._testcases_table` instance variable
+# and can be accessed by other modules using the `get` method.
+#
+# Copyright (C) 2019 Collabora Ltd
+# Luis Araujo <luis.araujo@collabora.co.uk>
+
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  US
+###################################################################################
+
+
+import os
+import copy
+import yaml
+import logging
+from subprocess import run, SubprocessError
+
+test_cases_repo = '/tmp/apertis-test-cases'
+test_cases_files = os.path.join(test_cases_repo, 'test-cases')
+
+priority_color = {
+    'low'      : 'secondary',
+    'medium'   : 'info',
+    'high'     : 'warning',
+    'critical' : 'danger'
+}
+
+
+class TestCasesTable(object):
+    """
+    This object represents the table for all the test cases from the
+    test case repository.
+    """
+
+    def __init__(self):
+        self._tmp_table = []
+        self._testcases_table = []
+        self.generate()
+
+    def set(self):
+        self._testcases_table = copy.deepcopy(self._tmp_table)
+
+    def get(self):
+        return self._testcases_table
+
+    def pull(self):
+        """Method to pull git repository updates"""
+        try:
+            cmd = run(['git', 'pull'], cwd=test_cases_repo)
+            if cmd.returncode != 0:
+                logging.error('git pull %s error: %d', test_cases_repo,
+                              cmd.returncode)
+        except SubprocessError as e:
+            logging.error(e)
+
+    def generate(self):
+        for root, _, files in os.walk(test_cases_files):
+            for f in files:
+                if f.endswith( ('.yml', '.yaml') ):
+                    self._make_table(os.path.join(root, f))
+        self.set()
+
+    def update(self):
+        self._tmp_table = []
+        self.pull()
+        self.generate()
+
+    def _make_table(self, tc_file):
+        try:
+            with open(tc_file, 'r', encoding='utf-8') as testcase:
+                tc_data = yaml.safe_load(testcase)
+        except yaml.scanner.ScannerError as e:
+            print("yaml format error:", e)
+            exit(1)
+
+        # Only automated tests are currently support, so skip manual tests.
+        # TODO: Manual tests should be added later.
+        if tc_data['metadata']['exec-type'] == 'manual':
+            return
+
+        tc_name = tc_data['metadata']['name']
+        image_type_platforms = []
+        for image_type , platforms in tc_data['metadata']['image-types'].items():
+            for platform in platforms:
+                # Apply this change to public jobs to keep consistency with the DB:
+                #    'minimal-armhf' => 'minimal-armhf-public'
+                v = platform.split('-')[-1]
+                if v != 'internal':
+                    platform += '-public'
+                image_type_platforms.append(image_type + '-' + platform)
+
+        image_deployments = [ e.lower() for e in
+                              tc_data['metadata']['image-deployment'] ]
+        self._tmp_table.append((tc_name,
+                                priority_color[tc_data['metadata']['priority']],
+                                image_deployments,
+                                image_type_platforms))

pages.py

@@ -23,7 +23,6 @@ import logging
 
 from flask import  request, render_template
 from models import Job, TestCase
-from data.tc_table import tc_table
 from config import config
 from sqlalchemy import desc
 
@@ -42,7 +41,8 @@ def generate_index(username=None):
                                                 key=lambda e: e[1],
                                                 reverse=True))
 
-def generate_report(image_release, image_version, image_deployment):
+def generate_report(image_release, image_version,
+                    image_deployment, testcases_table):
     logging.info('Generating report for release: %s , build: %s',
                  image_release, image_version)
 
@@ -82,7 +82,7 @@ def generate_report(image_release, image_version, image_deployment):
 
     return render_template('report.html',
                            test_cases=test_cases,
-                           tc_table=tc_table,
+                           tc_table=testcases_table,
                            test_cases_url=config['test-cases-url'],
                            image_urls=image_urls,
                            image_release=image_release,

run-qa-report

@@ -11,5 +11,11 @@ echo 'PostgreSQL Database is up'
 echo 'Running migration upgrades'
 python3 manage.py db upgrade
 
+# Copy test cases into /tmp directory so runtime user can access it
+if ! [ -e /tmp/apertis-test-cases ]; then
+    echo 'Copying test cases repo locally into /tmp ...'
+    cp -a /var/lib/apertis-test-cases /tmp/
+fi
+
 echo 'Starting QA Report App'
 ./webhook.py --config /app/secrets/config.yaml

webhook.py

@@ -25,6 +25,7 @@ import logging
 import argparse
 import datetime
 import os
+import atexit
 from collections import defaultdict
 
 from http import HTTPStatus
@@ -37,6 +38,9 @@ from models import db
 from save import save_job
 from pages import generate_index, generate_report
 
+from apscheduler.schedulers.background import BackgroundScheduler
+from data.testcases_table import TestCasesTable
+
 from flask import Flask, request, redirect, session, url_for
 from flask_dance.contrib.gitlab import make_gitlab_blueprint, gitlab
 from flask_dance.consumer import oauth_authorized
@@ -64,6 +68,13 @@ app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://%(user)s:%(pwd)s@%(host)s:
 app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
 db.init_app(app)
 
+# This object contains the main test cases table with methods to
+# keep it updated directly from the git repository.
+#
+# An initial tc table is created at object initialization and later will
+# be updated from the background.
+testcases_table = TestCasesTable()
+table_update_interval = 900
 
 @app.route('/login')
 def login():
@@ -85,7 +96,7 @@ def redirect_after_auth(blueprint, token):
 def get_report(image_release, image_version, image_deployment='apt'):
     try:
         return generate_report(image_release, image_version,
-                               image_deployment.lower())
+                               image_deployment.lower(), testcases_table.get())
     except Exception as e:
         logging.error(e)
         return 'Report error', HTTPStatus.NOT_FOUND
@@ -200,6 +211,10 @@ def series_complete(image_version):
     process_results(filtered_jobs, image_version)
     del finished[image_version]
 
+def update_testcases_table():
+    logging.info('Updating testcases_table data')
+    testcases_table.update()
+
 def logdata(datatype, data):
     try:
         logdir = '/tmp/lavaphabbridge-logs'
@@ -211,12 +226,26 @@ def logdata(datatype, data):
     except:
         pass
 
+@app.before_first_request
+def init_testcases_table_scheduler():
+    """
+    This function setups a scheduler background process to update the main
+    testcases_table structure from the apertis-test-cases repository at intervals
+    specified by `seconds`.
+    """
+    scheduler = BackgroundScheduler()
+    scheduler.add_job(func=update_testcases_table,
+                      trigger="interval",
+                      seconds=table_update_interval)
+    scheduler.start()
+    atexit.register(lambda: scheduler.shutdown())
+
 @app.after_request
 def log_response(response):
     if response.status_code >= 400:
         logging.warning('Returning error: {} "{}"'.format(response.status_code, response.get_data(as_text=True).split('\n')[0]))
     return response
-    
+
 def main():
     global tokens
     parser = argparse.ArgumentParser(description="Test report application")