diff --git a/format_template.yaml b/format_template.yaml
index e1828c87c6dbcd00862aef3e445101b4773951d5..cc3d646b306747402403b5c6ec570b37a936f020 100644
--- a/format_template.yaml
+++ b/format_template.yaml
@@ -31,6 +31,8 @@ metadata:
notes:
+ post-conditions:
+
# Only valid for automated non-ostree images installing packages.
install:
deps:
diff --git a/renderer/index.html b/renderer/index.html
index 1a4f0aeeb79fdf07718e6c7bd1deb5ddabe6fff3..88d74a14d87b3f44b774d33fb206ad847aa91560 100644
--- a/renderer/index.html
+++ b/renderer/index.html
@@ -1,4 +1,3 @@
-{% import 'macros.html' as macros %}
<!doctype html>
<html lang="en">
<head>
@@ -8,6 +7,7 @@
<title>{{ name }}</title>
</head>
<body>
+ {% import 'macros.html' as macros %}
<main role="main" class="container" style="margin-top: 40px; margin-bottom: 40px">
<h2>{{ name }} <small class="text-muted">{{ exec_type }}</small></h2>
<h3><span class="badge badge-{{ priority_color }}">{{ priority }}</span></h3>
@@ -97,7 +97,7 @@
</div>
</div>
{% endif %}
-
</main>
+
</body>
</html>
diff --git a/renderer/make_page.py b/renderer/make_page.py
index 0a20206cb258d4c6412e00dac732faf5be830e8a..99783843851b719dc13cbae9d48ca15914f18a06 100755
--- a/renderer/make_page.py
+++ b/renderer/make_page.py
@@ -7,6 +7,8 @@ import sys
import yaml
from jinja2 import Environment, FileSystemLoader
+from parser import parse_format, test_case_format
+
TEMPLATE_DIR="."
# This command parses each line of a list and returns a structure of the form
@@ -139,6 +141,8 @@ if '__main__' == __name__:
print("yaml format error:", e)
sys.exit(1)
+ parse_format(tc_data, test_case_format)
+
env = Environment(loader=FileSystemLoader([TEMPLATE_DIR]))
# Get template from environment and render it.
data = env.get_template('index.html').render(get_template_values(tc_data))
diff --git a/renderer/parser.py b/renderer/parser.py
new file mode 100644
index 0000000000000000000000000000000000000000..71b323f7ad14fa20575b8b5f0cced9c19e6dd474
--- /dev/null
+++ b/renderer/parser.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+#
+# This module defines the test case format and implement a
+# parser for such a format.
+#
+
+import sys
+import yaml
+
+
+test_case_format = {
+ 'metadata' : (True, {
+ 'name' : (True, ""),
+ 'format' : (False, ""),
+ 'image-type' : (True, [ 'target', 'minimal', 'ostree',
+ 'development', 'SDK', 'any' ]),
+ 'image-arch' : (True, ['amd64', 'arm64', 'armhf', 'any']),
+ 'type' : (True, ['functional', 'sanity', 'system']),
+ 'exec-type' : (True, ['manual' , 'automated']),
+ 'priority' : (True, ['low', 'medium', 'high', 'critical']),
+ 'maintainer' : (False, ""),
+ 'description' : (True, ""),
+ 'resources' : (False, []),
+ 'pre-conditions' : (False, []),
+ 'expected' : (True, []),
+ 'notes' : (False, []),
+ 'post-conditions' : (False, [])
+ }),
+ 'install' : (False, {
+ 'deps' : (True, [])
+ }),
+ 'run' : (True, {
+ 'steps' : (True, [])
+ }),
+ 'parse' : (False, {})
+}
+
+def parse_format(test_case, test_case_format):
+ for tagf, valuestr in test_case_format.items():
+ mandatory, valuef = valuestr
+ value = test_case.get(tagf)
+ if not value:
+ if mandatory:
+ sys.stderr.write("Error: mandatory field missing: '{}'\n"
+ .format(tagf))
+ sys.exit(1)
+ # Test case doesn't have this non-mandatory tag, so just continue.
+ continue
+
+ if type(value) == str and type(valuef) == list:
+ if tagf in ['image-type' , 'image-arch', 'type',
+ 'exec-type', 'priority']:
+ try:
+ valuef.index(value)
+ except ValueError:
+ sys.stderr.write("Warning: value '{}' not found in '{}' for "\
+ "field '{}'\n".format(value, valuef, tagf))
+ continue
+
+ if type(value) != type(valuef):
+ sys.stderr.write("Error: incorrect type for field: '{}'\n"
+ .format(tagf))
+ sys.exit(1)
+
+ if type(value) == dict:
+ parse_format(value, valuef)
+
+ return True
+
+
+if '__main__' == __name__:
+ testcase = sys.argv[1]
+ with open(testcase) as test_case:
+ test_case_data = yaml.safe_load(test_case)
+
+ parse_format(test_case_data, test_case_format)
diff --git a/tc/apparmor-utils.yaml b/tc/apparmor-utils.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..020f41c07e98a643c5aa0a09165f19583ddfad65
--- /dev/null
+++ b/tc/apparmor-utils.yaml
@@ -0,0 +1,57 @@
+metadata:
+ name: apparmor-utils
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: automated
+ priority: medium
+ maintainer: "Apertis Project"
+ description: "Test apparmor_parser and other tools in apparmor package
+ can be executed"
+
+ expected:
+ - "The test will show on stdout OK,FAIL or SKIP and exit code will be non zero
+ if at least one subtest will fail. A similar output will be shown:"
+ - |
+ >checking /bin/true is enforced: OK -
+ - "If the test_profile_syntax test-case in the apparmor-basic-profiles test
+ is failing, please report that failure instead: it produces better
+ diagnostics."
+ - "When that test case is failing, the go in complain mode, change for
+ enforced profiles in enforce->complain, change in complaininig profiles in
+ enforce->complain, gran total enforce/disable and change for enforced
+ profiles in enforce->disable test-cases in this test are also expected to
+ fail."
+
+ notes:
+ - "Make sure that you have disconnect the ethernet connection to the target
+ before you start the tethering process."
+ - "Implement a minimum set of test to be sure things works properly. No
+ advanced features tested."
+ - "This test depends on all AppArmor profiles being syntactically valid, and
+ does not have useful diagnostics if they are not. If the test_profile_syntax
+ test-case in the apparmor-basic-profiles test fails, please report that
+ failure instead. You can mention this failure in the same bug report, but
+ please do not report it separately."
+
+install:
+ deps:
+ - apparmor-utils-tests
+ - busybox
+ - apertis-tests-apparmor-report
+
+run:
+ steps:
+ - "# Run the the following commands:"
+ - echo -n | sudo tee /var/log/audit/audit.log
+ - 'common/run-test-in-systemd --name=aa-enforce-test --timeout 90 -- sh /usr/lib/apparmor-utils-tests/aa-enforce-test.sh'
+ - 'common/run-test-in-systemd --name=apparmor_parser --timeout 90 -- sh /usr/lib/apparmor-utils-tests/apparmor_parser.sh'
+ - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
+
+parse:
+ fixupdict:
+ FAIL: fail
+ OK: pass
+ SKIP: skip
+ pattern: ^(?P<test_case_id>.+):\s*(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)\s*-.*
diff --git a/tc/didcot.yaml b/tc/didcot.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..564180c13617d989cb55330295c4048884ab856c
--- /dev/null
+++ b/tc/didcot.yaml
@@ -0,0 +1,44 @@
+metadata:
+ name: didcot-service
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: unit
+ exec-type: automated
+ priority: medium
+ maintainer: "Apertis Project"
+
+ description: "Test the content hand-over feature by launching an application.
+ Didcot service is responsible for the content hand-over by
+ launching an application which supports the mime type or uri
+ schemes . The script launches a browser application which supports
+ http uri schemes"
+
+ expected:
+ - "The output should be similar to:"
+ - |
+ >Running test: /usr/share/installed-tests/didcot-0/didcot-test.test
+ # random seed: R02S69e7aa70c3f09ec156e00b6d49d5f4ec
+ 1..1
+ # Start of didcot-tests tests
+ ok 1 /didcot-tests/test_open_uri
+ # End of didcot-tests tests
+ PASS: didcot-0/didcot-test.test
+ SUMMARY: total: 1 passed: 1 skipped: 0 failed: 0
+
+install:
+ deps:
+ - gnome-desktop-testing
+ - didcot-tests
+
+run:
+ steps:
+ - "# Execute the following command:"
+ - common/run-test-in-systemd --user=user --timeout=900 --name=didcot gnome-desktop-testing-runner didcot
+
+parse:
+ fixupdict:
+ FAIL: fail
+ PASS: pass
+ SKIP: skip
+ pattern: ^(?P<result>PASS|FAIL|SKIP):\s*(?P<test_case_id>\S+)
diff --git a/tc/iptables-nmap.yaml b/tc/iptables-nmap.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..647a548c22f60c3d92c6cc6ad2caf64245c47622
--- /dev/null
+++ b/tc/iptables-nmap.yaml
@@ -0,0 +1,34 @@
+metadata:
+ name: iptables-nmap
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: manual
+ priority: high
+ maintainer: "Apertis Project"
+ description: "Test the firewall using nmap from another computer."
+
+ resources:
+ - "The tester needs an external computer with the nmap command available."
+ - "The external computer must be connected to the same network as the target."
+
+ expected:
+ - "All ports are filtered, except port 80/tcp (http) which is closed."
+ - |
+ >Not shown: 999 filtered ports
+ PORT STATE SERVICE
+ 80/tcp closed http
+
+ notes:
+ - "Make sure that you have disconnect the ethernet connection to the target
+ before you start the tethering process."
+ - "In order to test the SDK image, the VirtualBox VM must be configured with
+ a network attached to Bridged adaptor. The test cannot be run if the
+ network is configured as NAT."
+
+run:
+ steps:
+ - "From the external computer, check filtered/open/closed ports (the nmap
+ command can take some time):"
+ - "$ nmap <sac_ip>"
diff --git a/tc/webkit2gtk-gstreamer1.0.yaml b/tc/webkit2gtk-gstreamer1.0.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0ff8aa1c8f0e67065ce0494b0c9b7b4ce2bff798
--- /dev/null
+++ b/tc/webkit2gtk-gstreamer1.0.yaml
@@ -0,0 +1,22 @@
+metadata:
+ name: webkit2gtk-gstreamer1.0
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: manual
+ priority: medium
+ maintainer: "Apertis Project"
+ description: "Tests html5 video playback with GStreamer 1.0 in WebKit2GTK.
+ This test opens an URL to stream video and audio using
+ GtkClutterLauncher."
+
+ macro_install_packages_preconditions: webkit2gtk-testing
+
+ expected:
+ - "The video starts playing, with sound."
+
+run:
+ steps:
+ - "Launch the test application with the following URL running as normal user:"
+ - $ GtkClutterLauncher http://people.collabora.com/~em/wk/tests/video.html
diff --git a/tc/webkit2gtk-lazy-click.yaml b/tc/webkit2gtk-lazy-click.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..5f6a5f5e8b562d15c22adefcdc1dbbccf8b90e25
--- /dev/null
+++ b/tc/webkit2gtk-lazy-click.yaml
@@ -0,0 +1,45 @@
+metadata:
+ name: webkit2gtk-lazy-click
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: manual
+ priority: low
+ maintainer: "Apertis Project"
+ description: "Test touch adjustment (lazy click) support in WebKit2GTK."
+
+ resources:
+ - "A touch-screen."
+ - "A mouse."
+
+ macro_install_packages_preconditions: webkit2gtk-testing
+
+ expected:
+ - "Clicking with a mouse inside the green \"Link\" area should activate the
+ link."
+ - "Clicking with a mouse the red border outside the green \"Link\" area should
+ not activate the link."
+ - "Clicking with a mouse inside the blue \"Button\" area should activate the
+ button."
+ - "Clicking with a mouse the red border outside the blue \"Button\" area should
+ not activate the button."
+ - "Tapping with a single finger on the touch-screen inside the green \"Link\"
+ area should activate the link."
+ - "Tapping with a single finger on the touch-screen the red border outside
+ the green \"Link\" area should activate the link too."
+ - "Tapping with a single finger on the touch-screen inside the blue \"Button\"
+ area should activate the button."
+ - "Tapping with a single finger on the touch-screen the red border outside the
+ blue \"Button\" area should activate the button too."
+
+ notes:
+ - "Touching the very border of the red area may cause red to be printed and
+ that is not a bug. The area used by WebKit to search for related is round
+ and centered on the touched point, so it's hard to reproduce perfectly as
+ a border."
+
+run:
+ steps:
+ - "Run the test program:"
+ - $ GtkClutterLauncher file:///usr/share/webkit2gtk/testing/lazyclick.html
diff --git a/tools/format_checker.py b/tools/format_checker.py
deleted file mode 100755
index 1dc2bbe17ba0ae71961bec30ef642f8741fdee08..0000000000000000000000000000000000000000
--- a/tools/format_checker.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-#
-# Sanity check of the format for test cases files.
-#
-
-import sys
-import yaml
-
-FORMAT_TEMPLATE="format_template.yaml"
-
-if '__main__' == __name__:
-
- testcase = sys.argv[1]
-
- with open(FORMAT_TEMPLATE) as template:
- template_data = yaml.safe_load(template)
- #print(template_data)
-
- with open(testcase) as test_case:
- test_case_data = yaml.safe_load(test_case)
- print(test_case_data)
-
- test_case_keys = test_case_data.keys()
- if 'metadata' not in test_case_keys:
- print("ERROR: metadata not found: Test case should always have a "
- "metadata tag.")
- sys.exit(1)
- metadata_test_case_keys = test_case_data['metadata']
-
- # Template data
- template_keys = template_data.keys()
- metadata_template_keys = template_data['metadata']
-
- for tk in template_keys:
- if tk not in test_case_keys:
- print("INFO:", tk, "tag not found")
-
- for mtk in metadata_template_keys:
- if mtk not in metadata_test_case_keys:
- print("INFO:", mtk, "tag not found")