From a68258947da6de3f419e714217ef0ecf60050d61 Mon Sep 17 00:00:00 2001
From: Luis Araujo <luis.araujo@collabora.co.uk>
Date: Sat, 10 Nov 2018 00:28:04 +0800
Subject: [PATCH] Update test cases and add initial macro for preconditions
This commit updates the remaining test cases with the supported
format.
It also adds a first macro to conveniently add common pre-conditions
for ostree test cases.
Signed-off-by: Luis Araujo <luis.araujo@collabora.co.uk>
---
renderer/index.html | 35 +++++++++--------
renderer/macros.html | 13 +++++++
renderer/make_page.py | 32 ++++++++++++---
tc/eclipse-sysroot-management.yaml | 62 ++++++++++++++++++++++++++++++
tc/gettext-i18n.yaml | 13 +------
tc/gstreamer1.0-decode.yaml | 26 ++++++-------
tc/image-bootable.yaml | 18 ++++-----
tc/tumbler-thumbnailing.yaml | 39 ++++++++++---------
tc/webkit2gtk-mt-touch-events.yaml | 24 ++++++------
tc/x-hw-accelerated.yaml | 21 +++++-----
10 files changed, 188 insertions(+), 95 deletions(-)
create mode 100644 renderer/macros.html
create mode 100644 tc/eclipse-sysroot-management.yaml
diff --git a/renderer/index.html b/renderer/index.html
index 9baaa91..85e35bc 100644
--- a/renderer/index.html
+++ b/renderer/index.html
@@ -1,3 +1,4 @@
+{% import 'macros.html' as macros %}
<!doctype html>
<html lang="en">
<head>
@@ -9,7 +10,7 @@
<body>
<main role="main" class="container" style="margin-top: 40px; margin-bottom: 40px">
<h2>{{ name }} <small class="text-muted">{{ exec_type }}</small></h2>
- <h3><span class="badge badge-warning">{{ priority }}</span></h3>
+ <h3><span class="badge badge-{{ priority_color }}">{{ priority }}</span></h3>
<div class="card" style="margin-top: 20px">
<div class="card-body">
@@ -22,43 +23,46 @@
<dd class="col-sm-9">{{ type }}</dd>
</dl>
- <h3>Description</h3>
+ <h4>Description</h4>
<p>{{ description }}</p>
{% if resources %}
<hr />
- <h3>Resources</h3>
+ <h4>Resources</h4>
<ul>
{% for resource in resources %}
<li>{{ resource|e }}</li>
{% endfor %}
</ul>
{% endif %}
-
- {% if pre_conditions %}
+
+ {% if pre_conditions or pkgname %}
<hr />
- <h3>Pre Conditions</h3>
+ <h4>Pre Conditions</h4>
<ol>
- {% for cond, cmd in pre_conditions %}
- {% if cond %}<li>{{ cond|e }}</li>{% endif %}
- {% if cmd %}<p><kbd>{{ cmd|e }}</kbd></p>{% endif %}
+ {% if pkgname %}{{ macros.ostree_preconditions(pkgname) }}{% endif %}
+ {% for comment, command, output in pre_conditions %}
+ {% if comment %}<li class="mb-sm-2">{{ comment|e }}</li>{% endif %}
+ {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %}
+ {% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %}
{% endfor %}
</ol>
{% endif %}
<hr />
- <h3>Execution Steps</h3>
+ <h4>Execution Steps</h4>
<ol>
{% for comment, command in run_steps %}
- {% if comment %}<li>{{ comment|e }}</li>{% endif %}
+ {% if comment %}<li class="mb-sm-2">{{ comment|e }}</li>{% endif %}
{% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %}
{% endfor %}
</ol>
<hr />
<h4>Expected</h4>
- {% for comment, output in expected %}
+ {% for comment, command, output in expected %}
{% if comment %}<p class="mt-sm-3">{{ comment|e }}</p>{% endif %}
+ {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %}
{% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %}
{% endfor %}
</div>
@@ -69,9 +73,10 @@
<div class="card-body">
<h4>Notes</h4>
<ul>
- {% for note, cmd in notes %}
- {% if note %}<li>{{ note|e }}</li>{% endif %}
- {% if cmd %}<p><kbd>{{ cmd|e }}</kbd></p>{% endif %}
+ {% for comment, command, output in notes %}
+ {% if comment %}<li class="mb-sm-2">{{ comment|e }}</li>{% endif %}
+ {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %}
+ {% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %}
{% endfor %}
</ul>
</div>
diff --git a/renderer/macros.html b/renderer/macros.html
new file mode 100644
index 0000000..18dd992
--- /dev/null
+++ b/renderer/macros.html
@@ -0,0 +1,13 @@
+{% macro ostree_preconditions(pkgname) %}
+ <li class="mb-sm-2">From a PC, download and unpack the test data tarball from the gitlab test repository:</li>
+ <p><kbd>$ wget https://gitlab.apertis.org/tests/{{ pkgname }}/-/archive/master/{{ pkgname }}.tar.gz</kbd></p>
+ <p><kbd>$ tar -xvf {{ pkgname }}.tar.gz</kbd></p>
+ <li class="mb-sm-2">Copy the {{ pkgname }}-master-* to the device</li>
+ <p><kbd>$ DUT_IP=<device-ip></kbd></p>
+ <p><kbd>$ scp -r {{ pkgname }}-master-* user@$DUT_IP:</kbd></p>
+ <li class="mb-sm-2">Log into the target</li>
+ <p><kbd>$ ssh user@$DUT_IP</kbd></p>
+ <li class="mb-sm-2">After log into the DUT, enter the test directory</li>
+ <p><kbd>$ cd {{ pkgname }}-master-*</kbd></p>
+ <li class="mb-sm-2">Note that the tarball may change depending on the release/branch being tested, please make sure to download the correct tarball for the release in question.</li>
+{% endmacro %}
diff --git a/renderer/make_page.py b/renderer/make_page.py
index 1b20e18..cb72617 100755
--- a/renderer/make_page.py
+++ b/renderer/make_page.py
@@ -9,8 +9,17 @@ from jinja2 import Environment, FileSystemLoader
TEMPLATE_DIR="."
-# This command parse lists to make sure the following formatting
-# applies for the respective sections:
+
+def priority_color(priority):
+ return \
+ (priority == 'low' and 'secondary') or \
+ (priority == 'medium' and 'info') or \
+ (priority == 'high' and 'warning') or \
+ (priority == 'critical' and 'danger') or 'light'
+
+# This command parses each line of a list and returns a structure of the form
+# (comment, command, output) or (comment, command) to apply the following
+# formatting for the respective sections:
#
# pre-conditions:
# - Start line with '#' or '$' for commands, everything else is a comment.
@@ -19,8 +28,10 @@ TEMPLATE_DIR="."
# - Start line with '>' for command output, everything else is a comment.
#
# run: steps:
-# - Star '#' for comments , everything else is a command.
+# - Start line with '#' for comments , everything else is a command.
#
+# notes:
+# - Start line with '>' for command output, everything else is a comment.
def parse_list(lines, run=False):
processed_lines = []
for line in lines:
@@ -38,11 +49,11 @@ def parse_list(lines, run=False):
processed_lines.append(('', '$ '+sline))
else:
if sline[0] == '$':
- processed_lines.append(('', sline))
+ processed_lines.append(('', sline, ''))
elif sline[0] == '>':
- processed_lines.append(('', sline[1:].split('\n')))
+ processed_lines.append(('', '', sline[1:].split('\n')))
else:
- processed_lines.append((sline, ''))
+ processed_lines.append((sline, '', ''))
return processed_lines
@@ -62,6 +73,10 @@ def get_template_values(testcase_data):
if mv == 'expected':
template_values.update({ mv : parse_list(value) })
else:
+ # Set also the prority_color
+ if mv == 'priority':
+ template_values.update({ 'priority_color' :
+ priority_color(value) })
template_values.update({ mv.replace('-', '_') : value })
else:
print("Error: missing mandatory field", mv)
@@ -94,6 +109,11 @@ def get_template_values(testcase_data):
sys.exit(1)
template_values.update({ 'install_steps' : deps })
+ # Template macros
+ ostree_preconditions = metadata.get('ostree_preconditions')
+ if ostree_preconditions:
+ template_values.update({ 'pkgname' : ostree_preconditions })
+
return template_values
diff --git a/tc/eclipse-sysroot-management.yaml b/tc/eclipse-sysroot-management.yaml
new file mode 100644
index 0000000..d14edd7
--- /dev/null
+++ b/tc/eclipse-sysroot-management.yaml
@@ -0,0 +1,62 @@
+metadata:
+ name: eclipse-sysroot-management
+ format: "Apertis Test Definition 1.0"
+ image-type: SDK
+ image-arch: amd64
+ type: functional
+ exec-type: manual
+ priority: medium
+ description: "Test sysroot management and update within Eclipse.
+ It will go through configuring an existing sysroot in Eclipse,
+ and then updating it from the web with the latest version."
+ maintainer: "Apertis Project"
+
+ pre-conditions:
+ - "Ensure Rootfs is remounted as read/write."
+ - "$ sudo mount -o remount,rw /"
+ - "Install dependencies"
+ - "$ sudo apt install eclipse-cdt eclipse-plugins-apertis-management"
+ - "Restart the system to restore the filesystem state to read-only before
+ running the test."
+ - "$ sudo reboot"
+ - "There has to be a file available on the web with the name and URL of the
+ latest available sysroot, to allow Eclipse to check if there is a newer
+ version available. See the notes below."
+ - "A sysroot image is installed in /opt/sysroot/apertis"
+
+ expected:
+ - "Sysroot has been updated to latest version configured in the web file. To verify, check /opt/sysroot/apertis/binary/etc/image_version for the sysroot version."
+ - "Error log view should be empty."
+
+ notes:
+ - "Make sure that you have disconnect the ethernet connection to the target
+ before you start the tethering process."
+ - "The update process depends on a file available on the web that the plugin
+ uses to get information on the latest available version. You should make
+ this file available in a web server and point the 'Address to update' to it."
+ - "The format of the file is simple. Here is an example:"
+ - |
+ >version=16.12 20161019.0
+ url=https://images.apertis.org/sysroot/16.12/sysroot-apertis-16.12-armhf-development_20161019.0.tar.gz
+ - "This file and the images to download can be in different servers"
+ - "The images to use are the ones called
+ sysroot-AA.BB-apertis-development_XXXXXXXX.X.tar.gz"
+ - "For this test case be patient, it will take some time to download 600
+ Mbytes and install the sysroot."
+
+run:
+ steps:
+ - "# Start Eclipse"
+ - "# Go to Window > Preferences"
+ - "# Click on Apertis"
+ - "# Fill in Address with https://images.apertis.org/sysroot/16.12/sysroot"
+ - "# Uncheck Check SSL certificate errors"
+ - "# Click on Apply"
+ - "# Click on Update sysroot. The Update sysroot dialog should display the
+ current version and the latest available version on the web"
+ - "# Click on Ok. The dialog should close and the Progress view should display
+ lower right corner of eclipse main window. (This might takes a while
+ depends on bandwidth as the size of sysroot is around 600MB.)"
+ - "# Click on OK"
+ - "# Wait for the update job to complete"
+ - "# Check the error log view for any problem"
diff --git a/tc/gettext-i18n.yaml b/tc/gettext-i18n.yaml
index 4f24d93..8af8ad8 100644
--- a/tc/gettext-i18n.yaml
+++ b/tc/gettext-i18n.yaml
@@ -9,18 +9,7 @@ metadata:
maintainer: "Apertis Project"
description: "Check that gettext internationalization works."
- pre-conditions:
- - "From a PC, download and unpack the test data tarball from the gitlab test repository:"
- - "$ wget https://gitlab.apertis.org/tests/gettext-i18n/-/archive/master/gettext-i18n.tar.gz"
- - "$ tar -xvf gettext-i18n.tar.gz"
- - "Copy the gettext-i18n-master-* to the device"
- - "$ DUT_IP=<device-ip>"
- - "$ scp -r gettext-i18n-master-* user@$DUT_IP:"
- - "Log into the target"
- - "$ ssh user@$DUT_IP"
- - "After log into the DUT, enter the test directory"
- - "$ cd gettext-i18n-master-*"
- - "Note that the tarball may change depending on the release/branch being tested, please make sure to download the correct tarball for the release in question."
+ ostree_preconditions: gettext-i18n
expected:
- "The output should be pass or fail for each supported language."
diff --git a/tc/gstreamer1.0-decode.yaml b/tc/gstreamer1.0-decode.yaml
index c59b2b8..64bef2d 100644
--- a/tc/gstreamer1.0-decode.yaml
+++ b/tc/gstreamer1.0-decode.yaml
@@ -12,23 +12,23 @@ metadata:
required by any of the currently strategic functionalities."
pre-conditions:
- - "# From a PC, download and unpack the test data tarball from the gitlab test repository: https://gitlab.apertis.org/tests/gstreamer1.0-decode/-/archive/master/gstreamer1.0-decode-master.tar.gz"
- - wget https://gitlab.apertis.org/tests/gstreamer1.0-decode/-/archive/master/gstreamer1.0-decode-master.tar.gz
- - tar -xvf gstreamer1.0-decode-master.tar.gz
- - "# Copy the gstreamer1.0-decode-master-* to the device"
- - DUT_IP=<device-ip>
- - scp -r gstreamer1.0-decode-master-* user@$DUT_IP:
- - "# Log into the target"
- - ssh user@$DUT_IP
- - "# After log into the DUT, enter the test directory"
- - cd gstreamer1.0-decode-master-*
- - "# Note that the tarball may change depending on the release/branch being tested, please make sure to download the correct tarball for the release in question."
+ - "From a PC, download and unpack the test data tarball from the gitlab test repository:"
+ - "$ wget https://gitlab.apertis.org/tests/gstreamer1.0-decode/-/archive/master/gstreamer1.0-decode-master.tar.gz"
+ - "$ tar -xvf gstreamer1.0-decode-master.tar.gz"
+ - "Copy the to gstreamer1.0-decode-master-* the device"
+ - "$ DUT_IP=<device-ip>"
+ - "$ scp -r gstreamer1.0-decode-master-* user@$DUT_IP:"
+ - "Log into the target"
+ - "$ ssh user@$DUT_IP"
+ - "After log into the DUT, enter the test directory"
+ - "$ cd gstreamer1.0-decode-master-*"
+ - "Note that the tarball may change depending on the release/branch being tested, please make sure to download the correct tarball for the release in question."
expected:
- - "# The script will output a result for each media file it tries to decode. If it succeeds, PASSED will be displayed. If not, FAILED will be displayed."
+ - "The script will output a result for each media file it tries to decode. If it succeeds, PASSED will be displayed. If not, FAILED will be displayed."
notes:
- - "# If lava-test-shell test fails within lava job. It means e.g. a job has timed out."
+ - "If lava-test-shell test fails within lava job. It means e.g. a job has timed out."
run:
steps:
diff --git a/tc/image-bootable.yaml b/tc/image-bootable.yaml
index 5600a7a..b4e5fbc 100644
--- a/tc/image-bootable.yaml
+++ b/tc/image-bootable.yaml
@@ -11,17 +11,17 @@ metadata:
maintainer: "Apertis Project"
pre-conditions:
- - "# For SDK image: Remember, we're now requiring VirtualBox 4.2.2 for tests
- using Guest Additions. Please, read https://wiki.apertis.org/Docs/VirtualBox_Guest_AdditionsDocs/VirtualBox_Guest_Additions to update your setup."
- - "# Download the virtual machine image for the latest SDK release from
- https://images.apertis.org/"
- - "# For Target/Development images:"
- - "# Download latest target/development image released from
- https://images.apertis.org/"
- - "# Write image to a SD card (https://wiki.apertis.org/System_Image_Setup#Target_and_development_images"
+ - "For SDK image: Remember, we're now requiring VirtualBox 4.2.2 for tests
+ using Guest Additions. Please, read https://wiki.apertis.org/Docs/VirtualBox_Guest_AdditionsDocs/VirtualBox_Guest_Additions to update your setup."
+ - "Download the virtual machine image for the latest SDK release from
+ https://images.apertis.org/"
+ - "For Target/Development images:"
+ - "Download latest target/development image released from
+ https://images.apertis.org/"
+ - "Write image to a SD card (https://wiki.apertis.org/System_Image_Setup#Target_and_development_images"
expected:
- - "# Image boots either to a terminal or graphical environment."
+ - "Image boots either to a terminal or graphical environment."
run:
steps:
diff --git a/tc/tumbler-thumbnailing.yaml b/tc/tumbler-thumbnailing.yaml
index ce006df..fd7e33c 100644
--- a/tc/tumbler-thumbnailing.yaml
+++ b/tc/tumbler-thumbnailing.yaml
@@ -11,26 +11,28 @@ metadata:
maintainer: "Apertis Project"
pre-conditions:
- - "# Ensure Rootfs is remounted as read/write."
- - sudo mount -o remount,rw /
- - "# Install dependencies"
- - sudo apt install bash busybox dbus libglib2.0-bin procps tumbler
- - "# Restart the system to restore the filesystem state to read-only before
- running the test."
- - sudo reboot
+ - "Ensure Rootfs is remounted as read/write."
+ - "$ sudo mount -o remount,rw /"
+ - "Install dependencies"
+ - "$ sudo apt install bash busybox dbus libglib2.0-bin procps tumbler"
+ - "Restart the system to restore the filesystem state to read-only before
+ running the test."
+ - "$ sudo reboot"
expected:
- - "# The output should be similar to:"
- - ">>> Running test 'test_image_normal_thumbnail_generation' ...
- >>> Running test 'test_image_large_thumbnail_generation' ...
- >>> Running test 'test_document_normal_thumbnail_generation' ...
- >>> Running test 'test_document_large_thumbnail_generation' ...
- >>> Running test 'test_video_normal_thumbnail_generation' ...
- >>> Running test 'test_video_large_thumbnail_generation' ...
- >>> All tests PASSED successfully!"
- - "# If any test failed, they will be listed instead of the success message."
- - ">>> The following tests FAILED:
- [list of tests]"
+ - "The output should be similar to:"
+ - |
+ >>>> Running test 'test_image_normal_thumbnail_generation' ...
+ >>> Running test 'test_image_large_thumbnail_generation' ...
+ >>> Running test 'test_document_normal_thumbnail_generation' ...
+ >>> Running test 'test_document_large_thumbnail_generation' ...
+ >>> Running test 'test_video_normal_thumbnail_generation' ...
+ >>> Running test 'test_video_large_thumbnail_generation' ...
+ >>> All tests PASSED successfully!
+ - "If any test failed, they will be listed instead of the success message."
+ - |
+ >>>> The following tests FAILED:
+ [list of tests]
install:
deps:
@@ -43,6 +45,7 @@ install:
run:
steps:
+ - "# Run the test script:"
- common/run-test-in-systemd --timeout=900 --user=user --name=run-test env DEBUG=2 tumbler/automated/run-test.sh
parse:
diff --git a/tc/webkit2gtk-mt-touch-events.yaml b/tc/webkit2gtk-mt-touch-events.yaml
index 048ca46..9cedb51 100644
--- a/tc/webkit2gtk-mt-touch-events.yaml
+++ b/tc/webkit2gtk-mt-touch-events.yaml
@@ -13,20 +13,20 @@ metadata:
- "A touch-screen with multiple touch points."
pre-conditions:
- - "# Ensure Rootfs is remounted as read/write."
- - sudo mount -o remount,rw /
- - "# Install dependencies"
- - sudo apt install -o DPkg::options::="--force-unsafe-io" webkit2gtk-testing
- - "# Restart the system to restore the filesystem state to read-only before
- running the test."
- - sudo reboot
+ - "Ensure Rootfs is remounted as read/write."
+ - "$ sudo mount -o remount,rw /"
+ - "Install dependencies"
+ - $ sudo apt install -o DPkg::options::="--force-unsafe-io" webkit2gtk-testing
+ - "Restart the system to restore the filesystem state to read-only before
+ running the test."
+ - "$ sudo reboot"
expected:
- - "# Check that when sliding more than one finger within the canvas, lines of
- different colors are drawn in the \"Touch canvas\", each finger with
- its own."
- - "# Check that when pressing and dragging using the mouse, a line is drawn
- in the \"Mouse canvas\"."
+ - "Check that when sliding more than one finger within the canvas, lines of
+ different colors are drawn in the \"Touch canvas\", each finger with
+ its own."
+ - "Check that when pressing and dragging using the mouse, a line is drawn
+ in the \"Mouse canvas\"."
run:
steps:
diff --git a/tc/x-hw-accelerated.yaml b/tc/x-hw-accelerated.yaml
index 34b1ebc..4328655 100644
--- a/tc/x-hw-accelerated.yaml
+++ b/tc/x-hw-accelerated.yaml
@@ -10,18 +10,19 @@ metadata:
maintainer: "Apertis Project"
pre-conditions:
- - "# Ensure Rootfs is remounted as read/write."
- - sudo mount -o remount,rw /
- - "# Install dependencies"
- - sudo apt install mesa-utils-extra
- - "# Restart the system to restore the filesystem state to read-only before
- running the test."
- - sudo reboot
+ - "Ensure Rootfs is remounted as read/write."
+ - "$ sudo mount -o remount,rw /"
+ - "Install dependencies"
+ - "$ sudo apt install mesa-utils-extra"
+ - "Restart the system to restore the filesystem state to read-only before
+ running the test."
+ - "$ sudo reboot"
expected:
- - "# The command output should include the following information:"
- - "GL_VERSION: OpenGL ES 3.0 Mesa 10.1.0
- GL_RENDERER: Gallium 0.4 on llvmpipe (LLVM 3.4, 128 bits)"
+ - "The command output should include the following information:"
+ - |
+ >GL_VERSION: OpenGL ES 3.0 Mesa 10.1.0
+ GL_RENDERER: Gallium 0.4 on llvmpipe (LLVM 3.4, 128 bits)"
notes:
- "This will only test whether the correct renderer is used, actual
--
GitLab