Skip to content
Snippets Groups Projects
Commit 1c7dcb80 authored by Luis Araujo's avatar Luis Araujo
Browse files

Add parser for test case yaml files 


This commit addes a parser for the test case format, so it can
detect the most common mistakes (missing field or incorrect types
for values) in the test case yaml file.

This commit also adds a few more ported test cases.

Signed-off-by: default avatarLuis Araujo <luis.araujo@collabora.co.uk>
parent fc45841e
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 286 additions and 42 deletions
format_template.yaml
+ 2
− 0
View file @ 1c7dcb80
......@@ -31,6 +31,8 @@ metadata:
notes:
post-conditions:
# Only valid for automated non-ostree images installing packages.
install:
deps:
......
renderer/index.html
+ 2
− 2
View file @ 1c7dcb80
{% import 'macros.html' as macros %}
<!doctype html>
<html lang="en">
<head>
......@@ -8,6 +7,7 @@
<title>{{ name }}</title>
</head>
<body>
{% import 'macros.html' as macros %}
<main role="main" class="container" style="margin-top: 40px; margin-bottom: 40px">
<h2>{{ name }} <small class="text-muted">{{ exec_type }}</small></h2>
<h3><span class="badge badge-{{ priority_color }}">{{ priority }}</span></h3>
......@@ -97,7 +97,7 @@
</div>
</div>
{% endif %}
</main>
</body>
</html>
renderer/make_page.py
+ 4
− 0
View file @ 1c7dcb80
......@@ -7,6 +7,8 @@ import sys
import yaml
from jinja2 import Environment, FileSystemLoader
from parser import parse_format, test_case_format
TEMPLATE_DIR="."
# This command parses each line of a list and returns a structure of the form
......@@ -139,6 +141,8 @@ if '__main__' == __name__:
print("yaml format error:", e)
sys.exit(1)
parse_format(tc_data, test_case_format)
env = Environment(loader=FileSystemLoader([TEMPLATE_DIR]))
# Get template from environment and render it.
data = env.get_template('index.html').render(get_template_values(tc_data))
......
renderer/parser.py 0 → 100644
+ 76
− 0
View file @ 1c7dcb80
#!/usr/bin/env python3
#
# This module defines the test case format and implement a
# parser for such a format.
#
import sys
import yaml
test_case_format = {
'metadata' : (True, {
'name' : (True, ""),
'format' : (False, ""),
'image-type' : (True, [ 'target', 'minimal', 'ostree',
'development', 'SDK', 'any' ]),
'image-arch' : (True, ['amd64', 'arm64', 'armhf', 'any']),
'type' : (True, ['functional', 'sanity', 'system']),
'exec-type' : (True, ['manual' , 'automated']),
'priority' : (True, ['low', 'medium', 'high', 'critical']),
'maintainer' : (False, ""),
'description' : (True, ""),
'resources' : (False, []),
'pre-conditions' : (False, []),
'expected' : (True, []),
'notes' : (False, []),
'post-conditions' : (False, [])
}),
'install' : (False, {
'deps' : (True, [])
}),
'run' : (True, {
'steps' : (True, [])
}),
'parse' : (False, {})
}
def parse_format(test_case, test_case_format):
for tagf, valuestr in test_case_format.items():
mandatory, valuef = valuestr
value = test_case.get(tagf)
if not value:
if mandatory:
sys.stderr.write("Error: mandatory field missing: '{}'\n"
.format(tagf))
sys.exit(1)
# Test case doesn't have this non-mandatory tag, so just continue.
continue
if type(value) == str and type(valuef) == list:
if tagf in ['image-type' , 'image-arch', 'type',
'exec-type', 'priority']:
try:
valuef.index(value)
except ValueError:
sys.stderr.write("Warning: value '{}' not found in '{}' for "\
"field '{}'\n".format(value, valuef, tagf))
continue
if type(value) != type(valuef):
sys.stderr.write("Error: incorrect type for field: '{}'\n"
.format(tagf))
sys.exit(1)
if type(value) == dict:
parse_format(value, valuef)
return True
if '__main__' == __name__:
testcase = sys.argv[1]
with open(testcase) as test_case:
test_case_data = yaml.safe_load(test_case)
parse_format(test_case_data, test_case_format)
tc/apparmor-utils.yaml 0 → 100644
+ 57
− 0
View file @ 1c7dcb80
metadata:
name: apparmor-utils
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: functional
exec-type: automated
priority: medium
maintainer: "Apertis Project"
description: "Test apparmor_parser and other tools in apparmor package
can be executed"
expected:
- "The test will show on stdout OK,FAIL or SKIP and exit code will be non zero
if at least one subtest will fail. A similar output will be shown:"
- |
>checking /bin/true is enforced: OK -
- "If the test_profile_syntax test-case in the apparmor-basic-profiles test
is failing, please report that failure instead: it produces better
diagnostics."
- "When that test case is failing, the go in complain mode, change for
enforced profiles in enforce->complain, change in complaininig profiles in
enforce->complain, gran total enforce/disable and change for enforced
profiles in enforce->disable test-cases in this test are also expected to
fail."
notes:
- "Make sure that you have disconnect the ethernet connection to the target
before you start the tethering process."
- "Implement a minimum set of test to be sure things works properly. No
advanced features tested."
- "This test depends on all AppArmor profiles being syntactically valid, and
does not have useful diagnostics if they are not. If the test_profile_syntax
test-case in the apparmor-basic-profiles test fails, please report that
failure instead. You can mention this failure in the same bug report, but
please do not report it separately."
install:
deps:
- apparmor-utils-tests
- busybox
- apertis-tests-apparmor-report
run:
steps:
- "# Run the the following commands:"
- echo -n | sudo tee /var/log/audit/audit.log
- 'common/run-test-in-systemd --name=aa-enforce-test --timeout 90 -- sh /usr/lib/apparmor-utils-tests/aa-enforce-test.sh'
- 'common/run-test-in-systemd --name=apparmor_parser --timeout 90 -- sh /usr/lib/apparmor-utils-tests/apparmor_parser.sh'
- sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
parse:
fixupdict:
FAIL: fail
OK: pass
SKIP: skip
pattern: ^(?P<test_case_id>.+):\s*(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)\s*-.*
tc/didcot.yaml 0 → 100644
+ 44
− 0
View file @ 1c7dcb80
metadata:
name: didcot-service
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: unit
exec-type: automated
priority: medium
maintainer: "Apertis Project"
description: "Test the content hand-over feature by launching an application.
Didcot service is responsible for the content hand-over by
launching an application which supports the mime type or uri
schemes . The script launches a browser application which supports
http uri schemes"
expected:
- "The output should be similar to:"
- |
>Running test: /usr/share/installed-tests/didcot-0/didcot-test.test
# random seed: R02S69e7aa70c3f09ec156e00b6d49d5f4ec
1..1
# Start of didcot-tests tests
ok 1 /didcot-tests/test_open_uri
# End of didcot-tests tests
PASS: didcot-0/didcot-test.test
SUMMARY: total: 1 passed: 1 skipped: 0 failed: 0
install:
deps:
- gnome-desktop-testing
- didcot-tests
run:
steps:
- "# Execute the following command:"
- common/run-test-in-systemd --user=user --timeout=900 --name=didcot gnome-desktop-testing-runner didcot
parse:
fixupdict:
FAIL: fail
PASS: pass
SKIP: skip
pattern: ^(?P<result>PASS|FAIL|SKIP):\s*(?P<test_case_id>\S+)
tc/iptables-nmap.yaml 0 → 100644
+ 34
− 0
View file @ 1c7dcb80
metadata:
name: iptables-nmap
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: functional
exec-type: manual
priority: high
maintainer: "Apertis Project"
description: "Test the firewall using nmap from another computer."
resources:
- "The tester needs an external computer with the nmap command available."
- "The external computer must be connected to the same network as the target."
expected:
- "All ports are filtered, except port 80/tcp (http) which is closed."
- |
>Not shown: 999 filtered ports
PORT STATE SERVICE
80/tcp closed http
notes:
- "Make sure that you have disconnect the ethernet connection to the target
before you start the tethering process."
- "In order to test the SDK image, the VirtualBox VM must be configured with
a network attached to Bridged adaptor. The test cannot be run if the
network is configured as NAT."
run:
steps:
- "From the external computer, check filtered/open/closed ports (the nmap
command can take some time):"
- "$ nmap <sac_ip>"
tc/webkit2gtk-gstreamer1.0.yaml 0 → 100644
+ 22
− 0
View file @ 1c7dcb80
metadata:
name: webkit2gtk-gstreamer1.0
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: functional
exec-type: manual
priority: medium
maintainer: "Apertis Project"
description: "Tests html5 video playback with GStreamer 1.0 in WebKit2GTK.
This test opens an URL to stream video and audio using
GtkClutterLauncher."
macro_install_packages_preconditions: webkit2gtk-testing
expected:
- "The video starts playing, with sound."
run:
steps:
- "Launch the test application with the following URL running as normal user:"
- $ GtkClutterLauncher http://people.collabora.com/~em/wk/tests/video.html
tc/webkit2gtk-lazy-click.yaml 0 → 100644
+ 45
− 0
View file @ 1c7dcb80
metadata:
name: webkit2gtk-lazy-click
format: "Apertis Test Definition 1.0"
image-type: any
image-arch: any
type: functional
exec-type: manual
priority: low
maintainer: "Apertis Project"
description: "Test touch adjustment (lazy click) support in WebKit2GTK."
resources:
- "A touch-screen."
- "A mouse."
macro_install_packages_preconditions: webkit2gtk-testing
expected:
- "Clicking with a mouse inside the green \"Link\" area should activate the
link."
- "Clicking with a mouse the red border outside the green \"Link\" area should
not activate the link."
- "Clicking with a mouse inside the blue \"Button\" area should activate the
button."
- "Clicking with a mouse the red border outside the blue \"Button\" area should
not activate the button."
- "Tapping with a single finger on the touch-screen inside the green \"Link\"
area should activate the link."
- "Tapping with a single finger on the touch-screen the red border outside
the green \"Link\" area should activate the link too."
- "Tapping with a single finger on the touch-screen inside the blue \"Button\"
area should activate the button."
- "Tapping with a single finger on the touch-screen the red border outside the
blue \"Button\" area should activate the button too."
notes:
- "Touching the very border of the red area may cause red to be printed and
that is not a bug. The area used by WebKit to search for related is round
and centered on the touched point, so it's hard to reproduce perfectly as
a border."
run:
steps:
- "Run the test program:"
- $ GtkClutterLauncher file:///usr/share/webkit2gtk/testing/lazyclick.html
tools/format_checker.py deleted 100755 → 0
+ 0
− 40
View file @ fc45841e
#!/usr/bin/env python3
#
# Sanity check of the format for test cases files.
#
import sys
import yaml
FORMAT_TEMPLATE="format_template.yaml"
if '__main__' == __name__:
testcase = sys.argv[1]
with open(FORMAT_TEMPLATE) as template:
template_data = yaml.safe_load(template)
#print(template_data)
with open(testcase) as test_case:
test_case_data = yaml.safe_load(test_case)
print(test_case_data)
test_case_keys = test_case_data.keys()
if 'metadata' not in test_case_keys:
print("ERROR: metadata not found: Test case should always have a "
"metadata tag.")
sys.exit(1)
metadata_test_case_keys = test_case_data['metadata']
# Template data
template_keys = template_data.keys()
metadata_template_keys = template_data['metadata']
for tk in template_keys:
if tk not in test_case_keys:
print("INFO:", tk, "tag not found")
for mtk in metadata_template_keys:
if mtk not in metadata_test_case_keys:
print("INFO:", mtk, "tag not found")
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
Apertis Website Terms of Use Privacy Policy