Skip to content
Snippets Groups Projects
Commit 2d81d93d authored by Luis Araujo's avatar Luis Araujo
Browse files

WIP: Add some ported test cases and initial renderer code 


Signed-off-by: default avatarLuis Araujo <luis.araujo@collabora.co.uk>
parent 8a0202f3
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 592 additions and 18 deletions
format_definition.yaml
+ 2
− 2
View file @ 2d81d93d
......@@ -18,8 +18,8 @@ metadata:
# This can be used to define a series of test case types, and
# could be any of the following values:
# [ functional, sanity, system, unittest ]
type: unittest
# [ functional, sanity, system ]
type: functional
# The execution type of this test case:
# [ manual , automated ]
......
format_template.yaml
+ 14
− 3
View file @ 2d81d93d
#
# Formatting rules:
#
# pre-conditions:
# - Start line with '#' or '$' for commands, everything else is a comment.
#
# expected:
# - Start line with '>' for command output, everything else is a comment.
#
# run: steps:
# - Star '#' for comments , everything else is a command.
#
metadata:
name:
......@@ -10,7 +23,7 @@ metadata:
# [ amd64, arm64, armhf, any ]
image-arch:
# [ functional, sanity, system, unittest ]
# [ functional, sanity, system ]
type:
# [ manual , automated ]
......@@ -37,8 +50,6 @@ install:
run:
steps:
- "# Execute the following command."
- echo "Hello Apertis!"
# Parse test results from the test command output.
# This directive is only relevant for LAVA automated tests.
......
renderer/css/bootstrap.min.css 0 → 100644
+ 7
− 0
View file @ 2d81d93d
This diff is collapsed.
renderer/css/bootstrap.min.css.map 0 → 100644
+ 1
− 0
View file @ 2d81d93d
This diff is collapsed.
renderer/index.html 0 → 100644
+ 83
− 0
View file @ 2d81d93d
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<link href="css/bootstrap.min.css" rel="stylesheet">
<title>{{ name }}</title>
</head>
<body>
<main role="main" class="container" style="margin-top: 40px; margin-bottom: 40px">
<h2>{{ name }} <small class="text-muted">{{ exec_type }}</small></h2>
<h3><span class="badge badge-warning">{{ priority }}</span></h3>
<div class="card" style="margin-top: 20px">
<div class="card-body">
<dl class="row">
<dt class="col-sm-3">Image Type:</dt>
<dd class="col-sm-9">{{ image_type }}</dd>
<dt class="col-sm-3">Image Architecture:</dt>
<dd class="col-sm-9">{{ image_arch }}</dd>
<dt class="col-sm-3">Type:</dt>
<dd class="col-sm-9">{{ type }}</dd>
</dl>
<h3>Description</h3>
<p>{{ description }}</p>
{% if resources %}
<hr />
<h3>Resources</h3>
<ul>
{% for resource in resources %}
<li>{{ resource|e }}</li>
{% endfor %}
</ul>
{% endif %}
{% if pre_conditions %}
<hr />
<h3>Pre Conditions</h3>
<ol>
{% for cond, cmd in pre_conditions %}
{% if cond %}<li>{{ cond|e }}</li>{% endif %}
{% if cmd %}<p><kbd>{{ cmd|e }}</kbd></p>{% endif %}
{% endfor %}
</ol>
{% endif %}
<hr />
<h3>Execution Steps</h3>
<ol>
{% for comment, command in run_steps %}
{% if comment %}<li>{{ comment|e }}</li>{% endif %}
{% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %}
{% endfor %}
</ol>
<hr />
<h4>Expected</h4>
{% for comment, output in expected %}
{% if comment %}<p class="mt-sm-3">{{ comment|e }}</p>{% endif %}
{% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %}
{% endfor %}
</div>
</div>
{% if notes %}
<div class="card" style="margin-top: 30px">
<div class="card-body">
<h4>Notes</h4>
<ul>
{% for note, cmd in notes %}
{% if note %}<li>{{ note|e }}</li>{% endif %}
{% if cmd %}<p><kbd>{{ cmd|e }}</kbd></p>{% endif %}
{% endfor %}
</ul>
</div>
</div>
{% endif %}
</main>
</body>
</html>
renderer/make_page.py 0 → 100755
+ 119
− 0
View file @ 2d81d93d
#!/usr/bin/env python3
#
# Make a web page from a YAML test case file.
#
import sys
import yaml
from jinja2 import Environment, FileSystemLoader
TEMPLATE_DIR="."
# This command makes sure the following formatting applies for the respective
# sections:
#
# pre-conditions:
# - Start line with '#' or '$' for commands, everything else is a comment.
#
# expected:
# - Start line with '>' for command output, everything else is a comment.
#
# run: steps:
# - Star '#' for comments , everything else is a command.
#
def comments_commands(lines, run=False):
processed_lines = []
for line in lines:
p, c = '', ''
sline = line.strip()
if not sline:
processed_lines.append((p, c))
continue
if run:
if sline[0] == '#':
# Remove the '#' with the slice [1:]
processed_lines.append((sline[1:], ''))
else:
# Add the '$ ' in the line for the command
processed_lines.append(('', '$ '+sline))
else:
if sline[0] == '$':
processed_lines.append(('', sline))
elif sline[0] == '>':
processed_lines.append(('', sline[1:].split('\n')))
else:
processed_lines.append((sline, ''))
return processed_lines
def parse_format(testcase_data):
template_values = {}
# Mandatory fields
metadata = testcase_data.get('metadata')
if not metadata:
print("Error: missing mandatory field metadata")
sys.exit(1)
for mv in ['name', 'image-type', 'image-arch', 'type',
'exec-type', 'priority', 'description', 'expected']:
value = metadata.get(mv)
if value:
if mv == 'expected':
template_values.update({ mv : comments_commands(value) })
else:
template_values.update({ mv.replace('-', '_') : value })
else:
print("Error: missing mandatory field", mv)
sys.exit(1)
run = testcase_data.get('run')
if not run:
print("Error: missing mandatory field run")
sys.exit(1)
else:
steps = run.get('steps')
if not steps:
print("Error: missing mandatory field steps for run")
sys.exit(1)
template_values.update({ 'run_steps' :
comments_commands(steps, run=True) })
# No mandatory fields
for nm in ['notes', 'format', 'maintainer', 'resources', 'pre-conditions']:
value = metadata.get(nm)
if value:
template_values.update({ nm.replace('-', '_') :
comments_commands(value)
if nm in ['notes', 'pre-conditions']
else value })
install = testcase_data.get('install')
if install:
deps = install.get('deps')
if not deps:
print("Error: missing mandatory field deps for install")
sys.exit(1)
template_values.update({ 'install_steps' : deps })
return template_values
if '__main__' == __name__:
testcase_file = sys.argv[1]
try:
with open(testcase_file) as testcase:
testcase_data = yaml.safe_load(testcase)
except yaml.scanner.ScannerError as e:
print("yaml format error:", e)
sys.exit(1)
env = Environment(loader=FileSystemLoader([TEMPLATE_DIR]))
# Get template from environment and render it.
data = env.get_template('index.html').render(parse_format(testcase_data))
print(data)
tc/bluez-hfp.yaml 0 → 100644
+ 63
− 0
View file @ 2d81d93d
metadata:
name: bluez-hfp
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: functional
exec-type: manual
priority: medium
description: "Test Hands Free Profile (HFP) BlueZ feature."
maintainer: "Apertis Project"
resources:
- "A Bluetooth adapter (if you have two plugged, please remove one to avoid confusion)"
- "A Bluetooth-enabled phone with a SIM card able to make and receive calls."
pre-conditions:
- "Please note that connman disables bluetooth by default on a fresh image."
- "Enable device:"
- "$ connmanctl enable bluetooth"
- "Ensure Rootfs is remounted as read/write."
- "$ sudo mount -o remount,rw /"
- "Install dependencies"
- "$ sudo apt install chaiwala-tests python-dialog dialog python3-gi python3-dbus"
- "Restart the system to restore the filesystem state to read-only before
running the test."
- "$ sudo reboot"
expected:
- "If FAILED is displayed, one of the test has failed. The output should be similar to that:"
- |
>TEST STARTED: Pairing Initiatior
| Confiming passkey 805647 (/org/bluez/803/hci0/dev_78_47_1D_B3_6E_80)
TEST FINISHED: Pairing Initiatior : PASSED
TEST STARTED: Pairing Responder
| In the phone start pairing with BlueZ (00:1B:DC:0F:35:06).
| Device created /org/bluez/803/hci0/dev_78_47_1D_B3_6E_80
TEST FINISHED: Pairable Responder : PASSED
| Profiles supported:
* HFP AG
TEST STARTED: HFP AG
Type the phone number to call:*144
Press ENTER to hangup the call:
| Did you hear the call (y/n)
y
From a second phone call the phone connected to oFono.
Once the phone starts ring press ENTER to anwser:
Press ENTER to hangup the call:
| Did you hear the call (y/n)
y
TEST FINISHED: HFP AG : PASSED
notes:
- "There are some options to run the command."
- "The -i select which hci device to use(e.g.: hci0). One can find the hci device name by running 'hciconfig' and looking at the available devices in its outputs."
- "The -d select which device you want to pair with, skipping discovery procedure. One can find the phone's Bluetooth address by putting the phone in discoverable mode and using \"scan on\" command from bluetoothctl."
- "The -s skip pairing because the device is already paired. -d must be present when using this."
- "Example:"
- "$ /usr/lib/chaiwala-tests/bluez/bluez-hfp -i hci0 -d A0:F4:19:8E:D9:AF -s"
run:
steps:
- "# Execute the test suite:"
- DISPLAY=:0 /usr/lib/chaiwala-tests/bluez/bluez-hfp
tc/gettext-i18n.yaml
+ 13
− 13
View file @ 2d81d93d
......@@ -10,24 +10,24 @@ metadata:
description: "Check that gettext internationalization works."
pre-conditions:
- "# From a PC, download and unpack the test data tarball from the gitlab test repository: https://gitlab.apertis.org/tests/gettext-i18n/-/archive/master/gettext-i18n.tar.gz"
- wget https://gitlab.apertis.org/tests/gettext-i18n/-/archive/master/gettext-i18n.tar.gz
- tar -xvf gettext-i18n.tar.gz
- "# Copy the gettext-i18n-master-* to the device"
- DUT_IP=<device-ip>
- scp -r gettext-i18n-master-* user@$DUT_IP:
- "# Log into the target"
- ssh user@$DUT_IP
- "# After log into the DUT, enter the test directory"
- cd gettext-i18n-master-*
- "# Note that the tarball may change depending on the release/branch being tested, please make sure to download the correct tarball for the release in question."
- "From a PC, download and unpack the test data tarball from the gitlab test repository:"
- "$ wget https://gitlab.apertis.org/tests/gettext-i18n/-/archive/master/gettext-i18n.tar.gz"
- "$ tar -xvf gettext-i18n.tar.gz"
- "Copy the gettext-i18n-master-* to the device"
- "$ DUT_IP=<device-ip>"
- "$ scp -r gettext-i18n-master-* user@$DUT_IP:"
- "Log into the target"
- "$ ssh user@$DUT_IP"
- "After log into the DUT, enter the test directory"
- "$ cd gettext-i18n-master-*"
- "Note that the tarball may change depending on the release/branch being tested, please make sure to download the correct tarball for the release in question."
expected:
- "# The output should be pass or fail for each supported language."
- "The output should be pass or fail for each supported language."
run:
steps:
- "# Run the test script"
- "# Run the gettext test script:"
- common/run-test-in-systemd --timeout=900 --basename ./gettext-i18n.sh
parse:
......
tc/grilo.yaml 0 → 100644
+ 76
− 0
View file @ 2d81d93d
metadata:
name: grilo
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: functional
exec-type: automated
priority: medium
description: "Check grilo metadata notifications, and filesystem browsing."
maintainer: "Apertis Project"
pre-conditions:
- "Ensure Rootfs is remounted as read/write."
- "$ sudo mount -o remount,rw /"
- "Install dependencies"
- "$ sudo apt install apertis-tests gir1.2-grilo-0.2 grilo-plugins-0.2 python3 python3-gi"
- "Restart the system to restore the filesystem state to read-only before
running the test."
- "$ sudo reboot"
expected:
- "The output should be similar to this one:"
- |
>GriloTest: filesystem notifications
GriloTest: received 'content-changed' signal: file:///tmp/test-grilo-Y7UF7X added
GriloTest: received 'content-changed' signal: file:///tmp/test-grilo-Y7UF7X added
GriloTest: received 'content-changed' signal: file:///tmp/test-grilo-Y7UF7X/subdir added
GriloTest: received 'content-changed' signal: file:///tmp/test-grilo-Y7UF7X/subdir added
GriloTest: received 'content-changed' signal: file:///tmp/test-grilo-Y7UF7X changed
GriloTest: received 'content-changed' signal: file:///tmp/test-grilo-Y7UF7X/subdir removed
GriloTest: received 'content-changed' signal: file:///tmp/test-grilo-Y7UF7X/subdir removed
GriloTest: received 'content-changed' signal: file:///tmp/test-grilo-Y7UF7X removed
GriloTest: received 'content-changed' signal: file:///tmp/test-grilo-Y7UF7X removed
(python3:1426): Grilo-WARNING **: [filesystem] grl-filesystem.c:711: Got error: No such file or directory
ApertisTest: copying medias to /tmp/test-grilo-Y7UF7X
GriloTest: filesystem browse
GriloBrowserMixin: add media Videos/big_buck_bunny_smaller.ogv
GriloBrowserMixin: add media Pictures/320px-European_Common_Frog_Rana_temporaria.jpg
GriloBrowserMixin: add media Pictures/collabora-logo-big.png
GriloBrowserMixin: add media Music/Ghosts.pls
GriloBrowserMixin: add media Music/Generic_Sounds.pls
GriloBrowserMixin: add media Music/Ghosts.m3u
GriloBrowserMixin: add media Music/generic.oga
GriloBrowserMixin: add media Music/generic.wav
GriloBrowserMixin: add media Music/generic.mp3
GriloBrowserMixin: add media Music/generic.flac
GriloBrowserMixin: add media Music/generic-no-artwork.mp3
GriloTest: playlist
GriloBrowserMixin: add media Music/audio/generic.mp3
GriloBrowserMixin: add media Music/audio/generic.flac
GriloBrowserMixin: add media Music/audio/generic.oga
.
----------------------------------------------------------------------
Ran 1 test in 0.076s
OK
- "If any test failed, they will be listed instead of the success message:"
- |
>[python backtrace]
FAILED (errors=1)
install:
deps:
- apertis-tests
- gir1.2-grilo-0.2
- grilo-plugins-0.2
- python3
- python3-gi
run:
steps:
- "# Execute the following command:"
- common/run-test-in-systemd --timeout=900 --basename grilo/automated/test-grilo.py
parse:
pattern: ^(?P<test_case_id>[a-zA-Z0-9_\-\./]+):\s*(?P<result>pass|fail|skip|unknown)$
tc/image-bootable.yaml 0 → 100644
+ 28
− 0
View file @ 2d81d93d
metadata:
name: image-bootable
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: functional
exec-type: manual
priority: critical
description: "Test whether the image is bootable.
Basic functionality: booting up."
maintainer: "Apertis Project"
pre-conditions:
- "# For SDK image: Remember, we're now requiring VirtualBox 4.2.2 for tests
using Guest Additions. Please, read https://wiki.apertis.org/Docs/VirtualBox_Guest_AdditionsDocs/VirtualBox_Guest_Additions to update your setup."
- "# Download the virtual machine image for the latest SDK release from
https://images.apertis.org/"
- "# For Target/Development images:"
- "# Download latest target/development image released from
https://images.apertis.org/"
- "# Write image to a SD card (https://wiki.apertis.org/System_Image_Setup#Target_and_development_images"
expected:
- "# Image boots either to a terminal or graphical environment."
run:
steps:
- "# Boot image in target device or in the VM for the SDK image."
tc/image-gui-start.yaml 0 → 100644
+ 30
− 0
View file @ 2d81d93d
metadata:
name: image-gui-start
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: functional
exec-type: manual
priority: critical
description: "Test whether the image starts a graphical environment.
Basic functionality: ensuring that the GUI boots up."
maintainer: "Apertis Project"
pre-conditions:
- "For SDK image: Remember, we're now requiring VirtualBox 4.2.2 for tests
using Guest Additions. Please, read https://wiki.apertis.org/Docs/VirtualBox_Guest_AdditionsDocs/VirtualBox_Guest_Additions to update your setup."
- "Download the virtual machine image for the latest SDK release from
https://images.apertis.org/"
- "For Target/Development images:"
- "Download latest target/development image released from
https://images.apertis.org/"
- "Write image to a SD card as explained at: https://wiki.apertis.org/System_Image_Setup#Target_and_development_images"
expected:
- "The image correctly starts the respective graphical interface."
- "For SDK image: A XFCE environment comes up."
- "For Target/Development images: Image comes up with the Apertis HMI."
run:
steps:
- "# Boot image in target device or in the VM for the SDK image."
tc/tumbler-thumbnailing.yaml 0 → 100644
+ 49
− 0
View file @ 2d81d93d
metadata:
name: tumbler-thumbnailing
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: functional
exec-type: automated
priority: medium
description: "Check that all thumbnailing features of tumbler are working
properly (large, small, videos, documents, images)."
maintainer: "Apertis Project"
pre-conditions:
- "# Ensure Rootfs is remounted as read/write."
- sudo mount -o remount,rw /
- "# Install dependencies"
- sudo apt install bash busybox dbus libglib2.0-bin procps tumbler
- "# Restart the system to restore the filesystem state to read-only before
running the test."
- sudo reboot
expected:
- "# The output should be similar to:"
- ">>> Running test 'test_image_normal_thumbnail_generation' ...
>>> Running test 'test_image_large_thumbnail_generation' ...
>>> Running test 'test_document_normal_thumbnail_generation' ...
>>> Running test 'test_document_large_thumbnail_generation' ...
>>> Running test 'test_video_normal_thumbnail_generation' ...
>>> Running test 'test_video_large_thumbnail_generation' ...
>>> All tests PASSED successfully!"
- "# If any test failed, they will be listed instead of the success message."
- ">>> The following tests FAILED:
[list of tests]"
install:
deps:
- bash
- busybox
- dbus
- libglib2.0-bin
- procps
- tumbler
run:
steps:
- common/run-test-in-systemd --timeout=900 --user=user --name=run-test env DEBUG=2 tumbler/automated/run-test.sh
parse:
pattern: ^(?P<test_case_id>[a-zA-Z0-9_\-\./]+):\s*(?P<result>pass|fail|skip|unknown)$
tc/webkit2gtk-mt-touch-events.yaml 0 → 100644
+ 34
− 0
View file @ 2d81d93d
metadata:
name: webkit2gtk-mt-touch-events
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: functional
exec-type: manual
priority: low
description: "Test implementation of W3C touch events API in WebKit2GTK."
maintainer: "Apertis Project"
resources:
- "A touch-screen with multiple touch points."
pre-conditions:
- "# Ensure Rootfs is remounted as read/write."
- sudo mount -o remount,rw /
- "# Install dependencies"
- sudo apt install -o DPkg::options::="--force-unsafe-io" webkit2gtk-testing
- "# Restart the system to restore the filesystem state to read-only before
running the test."
- sudo reboot
expected:
- "# Check that when sliding more than one finger within the canvas, lines of
different colors are drawn in the \"Touch canvas\", each finger with
its own."
- "# Check that when pressing and dragging using the mouse, a line is drawn
in the \"Mouse canvas\"."
run:
steps:
- "# Launch the test application with one of the URL in resources:"
- GtkClutterLauncher file:///usr/share/webkit2gtk/testing/touchevents.html
tc/x-hw-accelerated.yaml 0 → 100644
+ 33
− 0
View file @ 2d81d93d
metadata:
name: x-hw-accelerated
format: "Apertis Test Definition 1.0"
image-type: sdk
image-arch: amd64
type: functional
exec-type: manual
priority: medium
description: "Test whether the correct GLES2 render is used."
maintainer: "Apertis Project"
pre-conditions:
- "# Ensure Rootfs is remounted as read/write."
- sudo mount -o remount,rw /
- "# Install dependencies"
- sudo apt install mesa-utils-extra
- "# Restart the system to restore the filesystem state to read-only before
running the test."
- sudo reboot
expected:
- "# The command output should include the following information:"
- "GL_VERSION: OpenGL ES 3.0 Mesa 10.1.0
GL_RENDERER: Gallium 0.4 on llvmpipe (LLVM 3.4, 128 bits)"
notes:
- "This will only test whether the correct renderer is used, actual
functionality is tested by clutter related tests."
run:
steps:
- "# Execute test command"
- es2_info
tools/format_checker.py 0 → 100755
+ 40
− 0
View file @ 2d81d93d
#!/usr/bin/env python3
#
# Sanity check of the format for test cases files.
#
import sys
import yaml
FORMAT_TEMPLATE="format_template.yaml"
if '__main__' == __name__:
testcase = sys.argv[1]
with open(FORMAT_TEMPLATE) as template:
template_data = yaml.safe_load(template)
#print(template_data)
with open(testcase) as test_case:
test_case_data = yaml.safe_load(test_case)
print(test_case_data)
test_case_keys = test_case_data.keys()
if 'metadata' not in test_case_keys:
print("ERROR: metadata not found: Test case should always have a "
"metadata tag.")
sys.exit(1)
metadata_test_case_keys = test_case_data['metadata']
# Template data
template_keys = template_data.keys()
metadata_template_keys = template_data['metadata']
for tk in template_keys:
if tk not in test_case_keys:
print("INFO:", tk, "tag not found")
for mtk in metadata_template_keys:
if mtk not in metadata_test_case_keys:
print("INFO:", mtk, "tag not found")
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
Apertis Website Terms of Use Privacy Policy