From 1c7dcb80057975ebc9b09ed68dc276e54f5a5dd8 Mon Sep 17 00:00:00 2001
From: Luis Araujo <luis.araujo@collabora.co.uk>
Date: Wed, 14 Nov 2018 00:55:31 +0800
Subject: [PATCH] Add parser for test case yaml files
This commit addes a parser for the test case format, so it can
detect the most common mistakes (missing field or incorrect types
for values) in the test case yaml file.
This commit also adds a few more ported test cases.
Signed-off-by: Luis Araujo <luis.araujo@collabora.co.uk>
---
format_template.yaml | 2 +
renderer/index.html | 4 +-
renderer/make_page.py | 4 ++
renderer/parser.py | 76 +++++++++++++++++++++++++++++++++
tc/apparmor-utils.yaml | 57 +++++++++++++++++++++++++
tc/didcot.yaml | 44 +++++++++++++++++++
tc/iptables-nmap.yaml | 34 +++++++++++++++
tc/webkit2gtk-gstreamer1.0.yaml | 22 ++++++++++
tc/webkit2gtk-lazy-click.yaml | 45 +++++++++++++++++++
tools/format_checker.py | 40 -----------------
10 files changed, 286 insertions(+), 42 deletions(-)
create mode 100644 renderer/parser.py
create mode 100644 tc/apparmor-utils.yaml
create mode 100644 tc/didcot.yaml
create mode 100644 tc/iptables-nmap.yaml
create mode 100644 tc/webkit2gtk-gstreamer1.0.yaml
create mode 100644 tc/webkit2gtk-lazy-click.yaml
delete mode 100755 tools/format_checker.py
diff --git a/format_template.yaml b/format_template.yaml
index e1828c8..cc3d646 100644
--- a/format_template.yaml
+++ b/format_template.yaml
@@ -31,6 +31,8 @@ metadata:
notes:
+ post-conditions:
+
# Only valid for automated non-ostree images installing packages.
install:
deps:
diff --git a/renderer/index.html b/renderer/index.html
index 1a4f0ae..88d74a1 100644
--- a/renderer/index.html
+++ b/renderer/index.html
@@ -1,4 +1,3 @@
-{% import 'macros.html' as macros %}
<!doctype html>
<html lang="en">
<head>
@@ -8,6 +7,7 @@
<title>{{ name }}</title>
</head>
<body>
+ {% import 'macros.html' as macros %}
<main role="main" class="container" style="margin-top: 40px; margin-bottom: 40px">
<h2>{{ name }} <small class="text-muted">{{ exec_type }}</small></h2>
<h3><span class="badge badge-{{ priority_color }}">{{ priority }}</span></h3>
@@ -97,7 +97,7 @@
</div>
</div>
{% endif %}
-
</main>
+
</body>
</html>
diff --git a/renderer/make_page.py b/renderer/make_page.py
index 0a20206..9978384 100755
--- a/renderer/make_page.py
+++ b/renderer/make_page.py
@@ -7,6 +7,8 @@ import sys
import yaml
from jinja2 import Environment, FileSystemLoader
+from parser import parse_format, test_case_format
+
TEMPLATE_DIR="."
# This command parses each line of a list and returns a structure of the form
@@ -139,6 +141,8 @@ if '__main__' == __name__:
print("yaml format error:", e)
sys.exit(1)
+ parse_format(tc_data, test_case_format)
+
env = Environment(loader=FileSystemLoader([TEMPLATE_DIR]))
# Get template from environment and render it.
data = env.get_template('index.html').render(get_template_values(tc_data))
diff --git a/renderer/parser.py b/renderer/parser.py
new file mode 100644
index 0000000..71b323f
--- /dev/null
+++ b/renderer/parser.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+#
+# This module defines the test case format and implement a
+# parser for such a format.
+#
+
+import sys
+import yaml
+
+
+test_case_format = {
+ 'metadata' : (True, {
+ 'name' : (True, ""),
+ 'format' : (False, ""),
+ 'image-type' : (True, [ 'target', 'minimal', 'ostree',
+ 'development', 'SDK', 'any' ]),
+ 'image-arch' : (True, ['amd64', 'arm64', 'armhf', 'any']),
+ 'type' : (True, ['functional', 'sanity', 'system']),
+ 'exec-type' : (True, ['manual' , 'automated']),
+ 'priority' : (True, ['low', 'medium', 'high', 'critical']),
+ 'maintainer' : (False, ""),
+ 'description' : (True, ""),
+ 'resources' : (False, []),
+ 'pre-conditions' : (False, []),
+ 'expected' : (True, []),
+ 'notes' : (False, []),
+ 'post-conditions' : (False, [])
+ }),
+ 'install' : (False, {
+ 'deps' : (True, [])
+ }),
+ 'run' : (True, {
+ 'steps' : (True, [])
+ }),
+ 'parse' : (False, {})
+}
+
+def parse_format(test_case, test_case_format):
+ for tagf, valuestr in test_case_format.items():
+ mandatory, valuef = valuestr
+ value = test_case.get(tagf)
+ if not value:
+ if mandatory:
+ sys.stderr.write("Error: mandatory field missing: '{}'\n"
+ .format(tagf))
+ sys.exit(1)
+ # Test case doesn't have this non-mandatory tag, so just continue.
+ continue
+
+ if type(value) == str and type(valuef) == list:
+ if tagf in ['image-type' , 'image-arch', 'type',
+ 'exec-type', 'priority']:
+ try:
+ valuef.index(value)
+ except ValueError:
+ sys.stderr.write("Warning: value '{}' not found in '{}' for "\
+ "field '{}'\n".format(value, valuef, tagf))
+ continue
+
+ if type(value) != type(valuef):
+ sys.stderr.write("Error: incorrect type for field: '{}'\n"
+ .format(tagf))
+ sys.exit(1)
+
+ if type(value) == dict:
+ parse_format(value, valuef)
+
+ return True
+
+
+if '__main__' == __name__:
+ testcase = sys.argv[1]
+ with open(testcase) as test_case:
+ test_case_data = yaml.safe_load(test_case)
+
+ parse_format(test_case_data, test_case_format)
diff --git a/tc/apparmor-utils.yaml b/tc/apparmor-utils.yaml
new file mode 100644
index 0000000..020f41c
--- /dev/null
+++ b/tc/apparmor-utils.yaml
@@ -0,0 +1,57 @@
+metadata:
+ name: apparmor-utils
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: automated
+ priority: medium
+ maintainer: "Apertis Project"
+ description: "Test apparmor_parser and other tools in apparmor package
+ can be executed"
+
+ expected:
+ - "The test will show on stdout OK,FAIL or SKIP and exit code will be non zero
+ if at least one subtest will fail. A similar output will be shown:"
+ - |
+ >checking /bin/true is enforced: OK -
+ - "If the test_profile_syntax test-case in the apparmor-basic-profiles test
+ is failing, please report that failure instead: it produces better
+ diagnostics."
+ - "When that test case is failing, the go in complain mode, change for
+ enforced profiles in enforce->complain, change in complaininig profiles in
+ enforce->complain, gran total enforce/disable and change for enforced
+ profiles in enforce->disable test-cases in this test are also expected to
+ fail."
+
+ notes:
+ - "Make sure that you have disconnect the ethernet connection to the target
+ before you start the tethering process."
+ - "Implement a minimum set of test to be sure things works properly. No
+ advanced features tested."
+ - "This test depends on all AppArmor profiles being syntactically valid, and
+ does not have useful diagnostics if they are not. If the test_profile_syntax
+ test-case in the apparmor-basic-profiles test fails, please report that
+ failure instead. You can mention this failure in the same bug report, but
+ please do not report it separately."
+
+install:
+ deps:
+ - apparmor-utils-tests
+ - busybox
+ - apertis-tests-apparmor-report
+
+run:
+ steps:
+ - "# Run the the following commands:"
+ - echo -n | sudo tee /var/log/audit/audit.log
+ - 'common/run-test-in-systemd --name=aa-enforce-test --timeout 90 -- sh /usr/lib/apparmor-utils-tests/aa-enforce-test.sh'
+ - 'common/run-test-in-systemd --name=apparmor_parser --timeout 90 -- sh /usr/lib/apparmor-utils-tests/apparmor_parser.sh'
+ - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
+
+parse:
+ fixupdict:
+ FAIL: fail
+ OK: pass
+ SKIP: skip
+ pattern: ^(?P<test_case_id>.+):\s*(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)\s*-.*
diff --git a/tc/didcot.yaml b/tc/didcot.yaml
new file mode 100644
index 0000000..564180c
--- /dev/null
+++ b/tc/didcot.yaml
@@ -0,0 +1,44 @@
+metadata:
+ name: didcot-service
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: unit
+ exec-type: automated
+ priority: medium
+ maintainer: "Apertis Project"
+
+ description: "Test the content hand-over feature by launching an application.
+ Didcot service is responsible for the content hand-over by
+ launching an application which supports the mime type or uri
+ schemes . The script launches a browser application which supports
+ http uri schemes"
+
+ expected:
+ - "The output should be similar to:"
+ - |
+ >Running test: /usr/share/installed-tests/didcot-0/didcot-test.test
+ # random seed: R02S69e7aa70c3f09ec156e00b6d49d5f4ec
+ 1..1
+ # Start of didcot-tests tests
+ ok 1 /didcot-tests/test_open_uri
+ # End of didcot-tests tests
+ PASS: didcot-0/didcot-test.test
+ SUMMARY: total: 1 passed: 1 skipped: 0 failed: 0
+
+install:
+ deps:
+ - gnome-desktop-testing
+ - didcot-tests
+
+run:
+ steps:
+ - "# Execute the following command:"
+ - common/run-test-in-systemd --user=user --timeout=900 --name=didcot gnome-desktop-testing-runner didcot
+
+parse:
+ fixupdict:
+ FAIL: fail
+ PASS: pass
+ SKIP: skip
+ pattern: ^(?P<result>PASS|FAIL|SKIP):\s*(?P<test_case_id>\S+)
diff --git a/tc/iptables-nmap.yaml b/tc/iptables-nmap.yaml
new file mode 100644
index 0000000..647a548
--- /dev/null
+++ b/tc/iptables-nmap.yaml
@@ -0,0 +1,34 @@
+metadata:
+ name: iptables-nmap
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: manual
+ priority: high
+ maintainer: "Apertis Project"
+ description: "Test the firewall using nmap from another computer."
+
+ resources:
+ - "The tester needs an external computer with the nmap command available."
+ - "The external computer must be connected to the same network as the target."
+
+ expected:
+ - "All ports are filtered, except port 80/tcp (http) which is closed."
+ - |
+ >Not shown: 999 filtered ports
+ PORT STATE SERVICE
+ 80/tcp closed http
+
+ notes:
+ - "Make sure that you have disconnect the ethernet connection to the target
+ before you start the tethering process."
+ - "In order to test the SDK image, the VirtualBox VM must be configured with
+ a network attached to Bridged adaptor. The test cannot be run if the
+ network is configured as NAT."
+
+run:
+ steps:
+ - "From the external computer, check filtered/open/closed ports (the nmap
+ command can take some time):"
+ - "$ nmap <sac_ip>"
diff --git a/tc/webkit2gtk-gstreamer1.0.yaml b/tc/webkit2gtk-gstreamer1.0.yaml
new file mode 100644
index 0000000..0ff8aa1
--- /dev/null
+++ b/tc/webkit2gtk-gstreamer1.0.yaml
@@ -0,0 +1,22 @@
+metadata:
+ name: webkit2gtk-gstreamer1.0
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: manual
+ priority: medium
+ maintainer: "Apertis Project"
+ description: "Tests html5 video playback with GStreamer 1.0 in WebKit2GTK.
+ This test opens an URL to stream video and audio using
+ GtkClutterLauncher."
+
+ macro_install_packages_preconditions: webkit2gtk-testing
+
+ expected:
+ - "The video starts playing, with sound."
+
+run:
+ steps:
+ - "Launch the test application with the following URL running as normal user:"
+ - $ GtkClutterLauncher http://people.collabora.com/~em/wk/tests/video.html
diff --git a/tc/webkit2gtk-lazy-click.yaml b/tc/webkit2gtk-lazy-click.yaml
new file mode 100644
index 0000000..5f6a5f5
--- /dev/null
+++ b/tc/webkit2gtk-lazy-click.yaml
@@ -0,0 +1,45 @@
+metadata:
+ name: webkit2gtk-lazy-click
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: manual
+ priority: low
+ maintainer: "Apertis Project"
+ description: "Test touch adjustment (lazy click) support in WebKit2GTK."
+
+ resources:
+ - "A touch-screen."
+ - "A mouse."
+
+ macro_install_packages_preconditions: webkit2gtk-testing
+
+ expected:
+ - "Clicking with a mouse inside the green \"Link\" area should activate the
+ link."
+ - "Clicking with a mouse the red border outside the green \"Link\" area should
+ not activate the link."
+ - "Clicking with a mouse inside the blue \"Button\" area should activate the
+ button."
+ - "Clicking with a mouse the red border outside the blue \"Button\" area should
+ not activate the button."
+ - "Tapping with a single finger on the touch-screen inside the green \"Link\"
+ area should activate the link."
+ - "Tapping with a single finger on the touch-screen the red border outside
+ the green \"Link\" area should activate the link too."
+ - "Tapping with a single finger on the touch-screen inside the blue \"Button\"
+ area should activate the button."
+ - "Tapping with a single finger on the touch-screen the red border outside the
+ blue \"Button\" area should activate the button too."
+
+ notes:
+ - "Touching the very border of the red area may cause red to be printed and
+ that is not a bug. The area used by WebKit to search for related is round
+ and centered on the touched point, so it's hard to reproduce perfectly as
+ a border."
+
+run:
+ steps:
+ - "Run the test program:"
+ - $ GtkClutterLauncher file:///usr/share/webkit2gtk/testing/lazyclick.html
diff --git a/tools/format_checker.py b/tools/format_checker.py
deleted file mode 100755
index 1dc2bbe..0000000
--- a/tools/format_checker.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-#
-# Sanity check of the format for test cases files.
-#
-
-import sys
-import yaml
-
-FORMAT_TEMPLATE="format_template.yaml"
-
-if '__main__' == __name__:
-
- testcase = sys.argv[1]
-
- with open(FORMAT_TEMPLATE) as template:
- template_data = yaml.safe_load(template)
- #print(template_data)
-
- with open(testcase) as test_case:
- test_case_data = yaml.safe_load(test_case)
- print(test_case_data)
-
- test_case_keys = test_case_data.keys()
- if 'metadata' not in test_case_keys:
- print("ERROR: metadata not found: Test case should always have a "
- "metadata tag.")
- sys.exit(1)
- metadata_test_case_keys = test_case_data['metadata']
-
- # Template data
- template_keys = template_data.keys()
- metadata_template_keys = template_data['metadata']
-
- for tk in template_keys:
- if tk not in test_case_keys:
- print("INFO:", tk, "tag not found")
-
- for mtk in metadata_template_keys:
- if mtk not in metadata_test_case_keys:
- print("INFO:", mtk, "tag not found")
--
GitLab