Update test cases and add initial macro for preconditions

This commit updates the remaining test cases with the supported format. It also adds a first macro to conveniently add common pre-conditions for ostree test cases. Signed-off-by: Luis Araujo <luis.araujo@collabora.co.uk>

Update test cases and add initial macro for preconditions
This commit updates the remaining test cases with the supported format. It also adds a first macro to conveniently add common pre-conditions for ostree test cases. Signed-off-by: Luis Araujo <luis.araujo@collabora.co.uk>
a6825894 · Luis Araujo · ab5b4c0f · a6825894 · a6825894 · a6825894
Commit a6825894 authored 6 years ago by Luis Araujo
--- a/renderer/index.html
+++ b/renderer/index.html
+{% import 'macros.html' as macros %}
 <!doctype html>
 <html lang="en">
  <head>
@@ -9,7 +10,7 @@
  <body>
    <main role="main" class="container" style="margin-top: 40px; margin-bottom: 40px">
      <h2>{{ name }} <small class="text-muted">{{ exec_type }}</small></h2>
-      <h3><span class="badge badge-warning">{{ priority }}</span></h3>
+      <h3><span class="badge badge-{{ priority_color }}">{{ priority }}</span></h3>
      <div class="card" style="margin-top: 20px">
 	<div class="card-body">
@@ -22,43 +23,46 @@
 	    <dd class="col-sm-9">{{ type }}</dd>
 	  </dl>
-      <h3>Description</h3>
+      <h4>Description</h4>
      <p>{{ description }}</p>
      {% if resources %}
      <hr />
-      <h3>Resources</h3>
+      <h4>Resources</h4>
      <ul>
      {% for resource in resources %}
      <li>{{ resource|e }}</li>
      {% endfor %}
      </ul>
      {% endif %}      
-      {% if pre_conditions %}
+      {% if pre_conditions or pkgname %}
      <hr />
-      <h3>Pre Conditions</h3>
+      <h4>Pre Conditions</h4>
      <ol>
-      {% for cond, cmd in pre_conditions %}
+      {% if pkgname %}{{ macros.ostree_preconditions(pkgname) }}{% endif %}
-      {% if cond %}<li>{{ cond|e }}</li>{% endif %}
+      {% for comment, command, output in pre_conditions %}
-      {% if cmd %}<p><kbd>{{ cmd|e }}</kbd></p>{% endif %}
+      {% if comment %}<li class="mb-sm-2">{{ comment|e }}</li>{% endif %}
+      {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %}
+      {% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %}
      {% endfor %}
      </ol>
      {% endif %}
      <hr />
-      <h3>Execution Steps</h3>
+      <h4>Execution Steps</h4>
      <ol>
      {% for comment, command in run_steps %}
-      {% if comment %}<li>{{ comment|e }}</li>{% endif %}
+      {% if comment %}<li class="mb-sm-2">{{ comment|e }}</li>{% endif %}
      {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %}
      {% endfor %}
      </ol>
      <hr />
      <h4>Expected</h4>
-      {% for comment, output in expected %}
+      {% for comment, command, output in expected %}
      {% if comment %}<p class="mt-sm-3">{{ comment|e }}</p>{% endif %}
+      {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %}
      {% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %}
      {% endfor %}
 	</div>
@@ -69,9 +73,10 @@
 	<div class="card-body">
      <h4>Notes</h4>
      <ul>
-      {% for note, cmd in notes %}
+      {% for comment, command, output in notes %}
-      {% if note %}<li>{{ note|e }}</li>{% endif %}
+      {% if comment %}<li class="mb-sm-2">{{ comment|e }}</li>{% endif %}
-      {% if cmd %}<p><kbd>{{ cmd|e }}</kbd></p>{% endif %}
+      {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %}
+      {% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %}
      {% endfor %}
      </ul>
        </div>

--- a/renderer/macros.html
+++ b/renderer/macros.html
+{% macro ostree_preconditions(pkgname) %}
+  <li class="mb-sm-2">From a PC, download and unpack the test data tarball from the gitlab test repository:</li>
+  <p><kbd>$ wget https://gitlab.apertis.org/tests/{{ pkgname }}/-/archive/master/{{ pkgname }}.tar.gz</kbd></p>
+  <p><kbd>$ tar -xvf {{ pkgname }}.tar.gz</kbd></p>
+  <li class="mb-sm-2">Copy the {{ pkgname }}-master-* to the device</li>
+  <p><kbd>$ DUT_IP=&ltdevice-ip&gt</kbd></p>
+  <p><kbd>$ scp -r {{ pkgname }}-master-* user@$DUT_IP:</kbd></p>
+  <li class="mb-sm-2">Log into the target</li>
+  <p><kbd>$ ssh user@$DUT_IP</kbd></p>
+  <li class="mb-sm-2">After log into the DUT, enter the test directory</li>
+  <p><kbd>$ cd {{ pkgname }}-master-*</kbd></p>
+  <li class="mb-sm-2">Note that the tarball may change depending on the release/branch being tested, please make sure to download the correct tarball for the release in question.</li>
+{% endmacro %}
--- a/renderer/make_page.py
+++ b/renderer/make_page.py
@@ -9,8 +9,17 @@ from jinja2 import Environment, FileSystemLoader
 TEMPLATE_DIR="."
-# This command parse lists to make sure the following formatting
-# applies for the respective sections:
+def priority_color(priority):
+    return \
+        (priority == 'low'      and 'secondary') or \
+        (priority == 'medium'   and 'info')      or \
+        (priority == 'high'     and 'warning')   or \
+        (priority == 'critical' and 'danger')    or 'light'
+# This command parses each line of a list and returns a structure of the form
+# (comment, command, output) or (comment, command) to apply the following
+# formatting for the respective sections:
 #
 # pre-conditions:
 #  - Start line with '#' or '$' for commands, everything else is a comment.
@@ -19,8 +28,10 @@ TEMPLATE_DIR="."
 #  - Start line with '>' for command output, everything else is a comment.
 #
 # run: steps:
-#  - Star '#' for comments , everything else is a command.
+#  - Start line with '#' for comments , everything else is a command.
 #
+# notes:
+#  - Start line with '>' for command output, everything else is a comment.
 def parse_list(lines, run=False):
    processed_lines = []
    for line in lines:        
@@ -38,11 +49,11 @@ def parse_list(lines, run=False):
                processed_lines.append(('', '$ '+sline))
        else:
            if sline[0] == '$':
-                processed_lines.append(('', sline))
+                processed_lines.append(('', sline, ''))
            elif sline[0] == '>':
-                processed_lines.append(('', sline[1:].split('\n')))
+                processed_lines.append(('', '', sline[1:].split('\n')))
            else:
-                processed_lines.append((sline, ''))
+                processed_lines.append((sline, '', ''))
    return processed_lines
@@ -62,6 +73,10 @@ def get_template_values(testcase_data):
            if mv == 'expected':
                template_values.update({ mv : parse_list(value) })
            else:
+                # Set also the prority_color
+                if mv == 'priority':
+                    template_values.update({ 'priority_color' :
+                                             priority_color(value) })
                template_values.update({ mv.replace('-', '_') : value })
        else:
            print("Error: missing mandatory field", mv)
@@ -94,6 +109,11 @@ def get_template_values(testcase_data):
            sys.exit(1)
        template_values.update({ 'install_steps' : deps })
+    # Template macros
+    ostree_preconditions = metadata.get('ostree_preconditions')
+    if ostree_preconditions:
+        template_values.update({ 'pkgname' : ostree_preconditions })
    return template_values

--- a/tc/eclipse-sysroot-management.yaml
+++ b/tc/eclipse-sysroot-management.yaml
+metadata:
+  name: eclipse-sysroot-management
+  format: "Apertis Test Definition 1.0"
+  image-type: SDK
+  image-arch: amd64
+  type: functional
+  exec-type: manual
+  priority: medium
+  description: "Test sysroot management and update within Eclipse.
+                It will go through configuring an existing sysroot in Eclipse,
+                and then updating it from the web with the latest version."
+  maintainer: "Apertis Project"
+  pre-conditions:
+    - "Ensure Rootfs is remounted as read/write."
+    - "$ sudo mount -o remount,rw /"
+    - "Install dependencies"
+    - "$ sudo apt install eclipse-cdt eclipse-plugins-apertis-management"
+    - "Restart the system to restore the filesystem state to read-only before
+       running the test."
+    - "$ sudo reboot"
+    - "There has to be a file available on the web with the name and URL of the
+       latest available sysroot, to allow Eclipse to check if there is a newer
+       version available. See the notes below."
+    - "A sysroot image is installed in /opt/sysroot/apertis"
+  expected:
+    - "Sysroot has been updated to latest version configured in the web file. To verify, check /opt/sysroot/apertis/binary/etc/image_version for the sysroot version."
+    - "Error log view should be empty."
+  notes:
+    - "Make sure that you have disconnect the ethernet connection to the target
+       before you start the tethering process."
+    - "The update process depends on a file available on the web that the plugin
+      uses to get information on the latest available version. You should make
+      this file available in a web server and point the 'Address to update' to it."
+    - "The format of the file is simple. Here is an example:"
+    - |
+        >version=16.12 20161019.0
+          url=https://images.apertis.org/sysroot/16.12/sysroot-apertis-16.12-armhf-development_20161019.0.tar.gz
+    - "This file and the images to download can be in different servers"
+    - "The images to use are the ones called
+       sysroot-AA.BB-apertis-development_XXXXXXXX.X.tar.gz"
+    - "For this test case be patient, it will take some time to download 600
+       Mbytes and install the sysroot."
+run:
+  steps:
+    - "# Start Eclipse"
+    - "# Go to Window > Preferences"
+    - "# Click on Apertis"
+    - "# Fill in Address with https://images.apertis.org/sysroot/16.12/sysroot"
+    - "# Uncheck Check SSL certificate errors"
+    - "# Click on Apply"
+    - "# Click on Update sysroot. The Update sysroot dialog should display the
+         current version and the latest available version on the web"
+    - "# Click on Ok. The dialog should close and the Progress view should display
+         lower right corner of eclipse main window. (This might takes a while
+	 depends on bandwidth as the size of sysroot is around 600MB.)"
+    - "# Click on OK"
+    - "# Wait for the update job to complete"
+    - "# Check the error log view for any problem"
--- a/tc/gettext-i18n.yaml
+++ b/tc/gettext-i18n.yaml
@@ -9,18 +9,7 @@ metadata:
  maintainer: "Apertis Project"
  description: "Check that gettext internationalization works."
-  pre-conditions:
+  ostree_preconditions: gettext-i18n
-    - "From a PC, download and unpack the test data tarball from the gitlab test repository:"
-    - "$ wget https://gitlab.apertis.org/tests/gettext-i18n/-/archive/master/gettext-i18n.tar.gz"
-    - "$ tar -xvf gettext-i18n.tar.gz"
-    - "Copy the gettext-i18n-master-* to the device"
-    - "$ DUT_IP=<device-ip>"
-    - "$ scp -r gettext-i18n-master-* user@$DUT_IP:"
-    - "Log into the target"
-    - "$ ssh user@$DUT_IP"
-    - "After log into the DUT, enter the test directory"
-    - "$ cd gettext-i18n-master-*"
-    - "Note that the tarball may change depending on the release/branch being tested, please make sure to download the correct tarball for the release in question."
  expected:
    - "The output should be pass or fail for each supported language."

--- a/tc/gstreamer1.0-decode.yaml
+++ b/tc/gstreamer1.0-decode.yaml
@@ -12,23 +12,23 @@ metadata:
                required by any of the currently strategic functionalities."
  pre-conditions:
-    - "# From a PC, download and unpack the test data tarball from the gitlab test repository: https://gitlab.apertis.org/tests/gstreamer1.0-decode/-/archive/master/gstreamer1.0-decode-master.tar.gz"
+    - "From a PC, download and unpack the test data tarball from the gitlab test repository:"
-    - wget https://gitlab.apertis.org/tests/gstreamer1.0-decode/-/archive/master/gstreamer1.0-decode-master.tar.gz
+    - "$ wget https://gitlab.apertis.org/tests/gstreamer1.0-decode/-/archive/master/gstreamer1.0-decode-master.tar.gz"
-    - tar -xvf gstreamer1.0-decode-master.tar.gz
+    - "$ tar -xvf gstreamer1.0-decode-master.tar.gz"
-    - "# Copy the gstreamer1.0-decode-master-* to the device"
+    - "Copy the to gstreamer1.0-decode-master-* the device"
-    - DUT_IP=<device-ip>
+    - "$ DUT_IP=<device-ip>"
-    - scp -r gstreamer1.0-decode-master-* user@$DUT_IP:
+    - "$ scp -r gstreamer1.0-decode-master-* user@$DUT_IP:"
-    - "# Log into the target"
+    - "Log into the target"
-    - ssh user@$DUT_IP
+    - "$ ssh user@$DUT_IP"
-    - "# After log into the DUT, enter the test directory"
+    - "After log into the DUT, enter the test directory"
-    - cd gstreamer1.0-decode-master-*
+    - "$ cd gstreamer1.0-decode-master-*"
-    - "# Note that the tarball may change depending on the release/branch being tested, please make sure to download the correct tarball for the release in question."
+    - "Note that the tarball may change depending on the release/branch being tested, please make sure to download the correct tarball for the release in question."
  expected:
-    - "# The script will output a result for each media file it tries to decode. If it succeeds, PASSED will be displayed. If not, FAILED will be displayed."
+    - "The script will output a result for each media file it tries to decode. If it succeeds, PASSED will be displayed. If not, FAILED will be displayed."
  notes:
-    - "# If lava-test-shell test fails within lava job. It means e.g. a job has timed out."
+    - "If lava-test-shell test fails within lava job. It means e.g. a job has timed out."
 run:
  steps:

--- a/tc/image-bootable.yaml
+++ b/tc/image-bootable.yaml
@@ -11,17 +11,17 @@ metadata:
  maintainer: "Apertis Project"
  pre-conditions:
-    - "# For SDK image: Remember, we're now requiring VirtualBox 4.2.2 for tests
+    - "For SDK image: Remember, we're now requiring VirtualBox 4.2.2 for tests
-         using Guest Additions. Please, read https://wiki.apertis.org/Docs/VirtualBox_Guest_AdditionsDocs/VirtualBox_Guest_Additions to update your setup."
+       using Guest Additions. Please, read https://wiki.apertis.org/Docs/VirtualBox_Guest_AdditionsDocs/VirtualBox_Guest_Additions to update your setup."
-    - "# Download the virtual machine image for the latest SDK release from
+    - "Download the virtual machine image for the latest SDK release from
-         https://images.apertis.org/"
+       https://images.apertis.org/"
-    - "# For Target/Development images:"
+    - "For Target/Development images:"
-    - "# Download latest target/development image released from
+    - "Download latest target/development image released from
-         https://images.apertis.org/"
+       https://images.apertis.org/"
-    - "# Write image to a SD card (https://wiki.apertis.org/System_Image_Setup#Target_and_development_images"
+    - "Write image to a SD card (https://wiki.apertis.org/System_Image_Setup#Target_and_development_images"
  expected:
-    - "# Image boots either to a terminal or graphical environment."
+    - "Image boots either to a terminal or graphical environment."
 run:
  steps:

--- a/tc/tumbler-thumbnailing.yaml
+++ b/tc/tumbler-thumbnailing.yaml
@@ -11,26 +11,28 @@ metadata:
  maintainer: "Apertis Project"
  pre-conditions:
-    - "# Ensure Rootfs is remounted as read/write."
+    - "Ensure Rootfs is remounted as read/write."
-    - sudo mount -o remount,rw /
+    - "$ sudo mount -o remount,rw /"
-    - "# Install dependencies"
+    - "Install dependencies"
-    - sudo apt install bash busybox dbus libglib2.0-bin procps tumbler
+    - "$ sudo apt install bash busybox dbus libglib2.0-bin procps tumbler"
-    - "# Restart the system to restore the filesystem state to read-only before
+    - "Restart the system to restore the filesystem state to read-only before
-         running the test."
+       running the test."
-    - sudo reboot
+    - "$ sudo reboot"
  expected:
-    - "# The output should be similar to:"
+    - "The output should be similar to:"
-    - ">>> Running test 'test_image_normal_thumbnail_generation' ...
+    - |
-       >>> Running test 'test_image_large_thumbnail_generation' ...
+        >>>> Running test 'test_image_normal_thumbnail_generation' ...
-       >>> Running test 'test_document_normal_thumbnail_generation' ...
+        >>> Running test 'test_image_large_thumbnail_generation' ...
-       >>> Running test 'test_document_large_thumbnail_generation' ...
+        >>> Running test 'test_document_normal_thumbnail_generation' ...
-       >>> Running test 'test_video_normal_thumbnail_generation' ...
+        >>> Running test 'test_document_large_thumbnail_generation' ...
-       >>> Running test 'test_video_large_thumbnail_generation' ...
+        >>> Running test 'test_video_normal_thumbnail_generation' ...
-       >>> All tests PASSED successfully!"
+        >>> Running test 'test_video_large_thumbnail_generation' ...
-    - "# If any test failed, they will be listed instead of the success message."
+        >>> All tests PASSED successfully!
-    - ">>> The following tests FAILED:
+    - "If any test failed, they will be listed instead of the success message."
-       [list of tests]"
+    - |
+        >>>> The following tests FAILED:
+        [list of tests]
 install:
  deps:
@@ -43,6 +45,7 @@ install:
 run:
  steps:
+    - "# Run the test script:"
    - common/run-test-in-systemd --timeout=900 --user=user --name=run-test env DEBUG=2 tumbler/automated/run-test.sh
 parse:

--- a/tc/webkit2gtk-mt-touch-events.yaml
+++ b/tc/webkit2gtk-mt-touch-events.yaml
@@ -13,20 +13,20 @@ metadata:
    - "A touch-screen with multiple touch points."
  pre-conditions:
-    - "# Ensure Rootfs is remounted as read/write."
+    - "Ensure Rootfs is remounted as read/write."
-    - sudo mount -o remount,rw /
+    - "$ sudo mount -o remount,rw /"
-    - "# Install dependencies"
+    - "Install dependencies"
-    - sudo apt install -o DPkg::options::="--force-unsafe-io" webkit2gtk-testing
+    - $ sudo apt install -o DPkg::options::="--force-unsafe-io" webkit2gtk-testing
-    - "# Restart the system to restore the filesystem state to read-only before
+    - "Restart the system to restore the filesystem state to read-only before
-         running the test."
+       running the test."
-    - sudo reboot
+    - "$ sudo reboot"
  expected:
-    - "# Check that when sliding more than one finger within the canvas, lines of
+    - "Check that when sliding more than one finger within the canvas, lines of
-         different colors are drawn in the \"Touch canvas\", each finger with
+       different colors are drawn in the \"Touch canvas\", each finger with
-         its own."
+       its own."
-    - "# Check that when pressing and dragging using the mouse, a line is drawn
+    - "Check that when pressing and dragging using the mouse, a line is drawn
-         in the \"Mouse canvas\"."
+       in the \"Mouse canvas\"."
 run:
  steps:

--- a/tc/x-hw-accelerated.yaml
+++ b/tc/x-hw-accelerated.yaml
@@ -10,18 +10,19 @@ metadata:
  maintainer: "Apertis Project"
  pre-conditions:
-    - "# Ensure Rootfs is remounted as read/write."
+    - "Ensure Rootfs is remounted as read/write."
-    - sudo mount -o remount,rw /
+    - "$ sudo mount -o remount,rw /"
-    - "# Install dependencies"
+    - "Install dependencies"
-    - sudo apt install mesa-utils-extra
+    - "$ sudo apt install mesa-utils-extra"
-    - "# Restart the system to restore the filesystem state to read-only before
+    - "Restart the system to restore the filesystem state to read-only before
-         running the test."
+       running the test."
-    - sudo reboot
+    - "$ sudo reboot"
  expected:
-    - "# The command output should include the following information:"
+    - "The command output should include the following information:"
-    - "GL_VERSION: OpenGL ES 3.0 Mesa 10.1.0
+    - |
-       GL_RENDERER: Gallium 0.4 on llvmpipe (LLVM 3.4, 128 bits)"
+        >GL_VERSION: OpenGL ES 3.0 Mesa 10.1.0
+        GL_RENDERER: Gallium 0.4 on llvmpipe (LLVM 3.4, 128 bits)"
  notes:
    - "This will only test whether the correct renderer is used, actual