From c78e403e96f05f3bc2b95647f447befcfe8c79a8 Mon Sep 17 00:00:00 2001
From: Luis Araujo <luis.araujo@collabora.co.uk>
Date: Thu, 15 Nov 2018 01:51:17 +0800
Subject: [PATCH] Add command line interface to the renderer
- Adds a command line interface to the renderer to process a single
test case yaml file or a directory of files.
- Adds the feature to write the test case html page inside a specific
directory.
- Port few more test cases to the yaml format.
Signed-off-by: Luis Araujo <luis.araujo@collabora.co.uk>
---
renderer/make_page.py | 53 ++++++++++++++++++++---
renderer/{ => templates}/index.html | 2 +-
renderer/{ => templates}/macros.html | 0
tc/apparmor-tracker.yaml | 59 ++++++++++++++++++++++++++
tc/iptables-basic.yaml | 36 ++++++++++++++++
tc/tracker-indexing-local-storage.yaml | 2 +-
tc/traffic-control-basic.yaml | 52 +++++++++++++++++++++++
tc/x-hw-accelerated.yaml | 2 +-
8 files changed, 196 insertions(+), 10 deletions(-)
rename renderer/{ => templates}/index.html (98%)
rename renderer/{ => templates}/macros.html (100%)
create mode 100644 tc/apparmor-tracker.yaml
create mode 100644 tc/iptables-basic.yaml
create mode 100644 tc/traffic-control-basic.yaml
diff --git a/renderer/make_page.py b/renderer/make_page.py
index 9978384..299d217 100755
--- a/renderer/make_page.py
+++ b/renderer/make_page.py
@@ -3,9 +3,12 @@
# Make a web page from a YAML test case file.
#
+import os
import sys
import yaml
+import shutil
from jinja2 import Environment, FileSystemLoader
+from argparse import ArgumentParser
from parser import parse_format, test_case_format
@@ -130,21 +133,57 @@ def get_template_values(testcase_data):
return template_values
-
-if '__main__' == __name__:
- testcase_file = sys.argv[1]
-
+def generate_test_case(tc_file, directory):
try:
- with open(testcase_file) as testcase:
+ with open(tc_file) as testcase:
tc_data = yaml.safe_load(testcase)
except yaml.scanner.ScannerError as e:
print("yaml format error:", e)
sys.exit(1)
+ # Parse file to detect any syntax error.
+ print("Parsing file", tc_file)
parse_format(tc_data, test_case_format)
env = Environment(loader=FileSystemLoader([TEMPLATE_DIR]))
# Get template from environment and render it.
- data = env.get_template('index.html').render(get_template_values(tc_data))
+ data = env.get_template('templates/index.html').render(get_template_values(tc_data))
+
+ filename = os.path.splitext(os.path.basename(tc_file))[0] + ".html"
+ print("Generating test case page", filename)
+ with open(os.path.join(directory, filename), 'w') as html_file:
+ html_file.write(data)
+
- print(data)
+if '__main__' == __name__:
+ cli_parser = ArgumentParser(description="make_page")
+ cli_parser.add_argument('-d', '--test-case-dir',
+ help="Directory path for generated test cases")
+ cli_parser.add_argument('yaml_files',
+ help="YAML file or files directory")
+ args = cli_parser.parse_args()
+
+ directory = '.'
+ if args.test_case_dir:
+ directory = args.test_case_dir
+ try:
+ os.mkdir(directory)
+ except FileExistsError:
+ print("Directory '{}' already exists".format(directory))
+ except e:
+ print("Error:", e)
+ sys.exit(1)
+
+ if os.path.isfile(args.yaml_files):
+ generate_test_case(args.yaml_files, directory)
+ else:
+ for root, _, files in os.walk(args.yaml_files):
+ for f in files:
+ tc_file = os.path.join(root, f)
+ if os.path.isfile(tc_file):
+ generate_test_case(tc_file, directory)
+
+ cssdir = os.path.join(directory, 'css')
+ if not (os.path.exists(cssdir) and os.path.isdir(cssdir)):
+ print("Copying css style to", cssdir)
+ shutil.copytree('css/', cssdir)
diff --git a/renderer/index.html b/renderer/templates/index.html
similarity index 98%
rename from renderer/index.html
rename to renderer/templates/index.html
index 88d74a1..3207e39 100644
--- a/renderer/index.html
+++ b/renderer/templates/index.html
@@ -7,7 +7,7 @@
<title>{{ name }}</title>
</head>
<body>
- {% import 'macros.html' as macros %}
+ {% import 'templates/macros.html' as macros %}
<main role="main" class="container" style="margin-top: 40px; margin-bottom: 40px">
<h2>{{ name }} <small class="text-muted">{{ exec_type }}</small></h2>
<h3><span class="badge badge-{{ priority_color }}">{{ priority }}</span></h3>
diff --git a/renderer/macros.html b/renderer/templates/macros.html
similarity index 100%
rename from renderer/macros.html
rename to renderer/templates/macros.html
diff --git a/tc/apparmor-tracker.yaml b/tc/apparmor-tracker.yaml
new file mode 100644
index 0000000..a136177
--- /dev/null
+++ b/tc/apparmor-tracker.yaml
@@ -0,0 +1,59 @@
+metadata:
+ name: apparmor-tracker
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: automated
+ priority: medium
+ maintainer: "Apertis Project"
+ description: "Tests that the tracker abstraction and tracker profiles don't
+ cause false negatives and that it does not allow arbitrary reading
+ of files in the home directory."
+
+ pre-conditions:
+ - "Download the media tarball:"
+ - $ wget https://people.collabora.com/~araujo/apertis-test-media.tar.bz2
+ - "Create directory and unpack media in the shared folder."
+ - $ mkdir -p /home/shared/
+ - $ tar -jxvf apertis-test-media.tar.bz2 -C /home/shared/
+
+ expected:
+ - "Both tests should report pass and not fail:"
+ - |
+ >tracker.normal.expected: pass
+ tracker.malicious.expected: pass
+
+ notes:
+ - "Make sure that you have disconnect the ethernet connection to the target
+ before you start the tethering process."
+ - "The results of the tests printed to the standard output are enough to
+ determine whether they passed or fail."
+ - "If you get a failure and need to report a bug please run sudo watch-aa on
+ a terminal and run the case that is failing (separately, if both), and add
+ the logs you get to the bug report."
+
+install:
+ deps:
+ - busybox
+ - apertis-tests-apparmor-report
+ - apertis-tests-apparmor-tracker
+ - gir1.2-grilo-0.2
+ - gir1.2-tracker-1.0
+ - python3
+ - python3-gi
+ - wget
+ steps:
+ - wget https://people.collabora.com/~araujo/apertis-test-media.tar.bz2
+ - mkdir -p /home/shared/
+ - tar -jxvf apertis-test-media.tar.bz2 -C /home/shared/
+
+run:
+ steps:
+ - "# Execute the following commands:"
+ - echo -n | sudo tee /var/log/audit/audit.log
+ - common/run-test-in-systemd --name run-test-tracker --timeout 3000 apparmor/tracker/test-tracker
+ - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
+
+parse:
+ pattern: ^(?P<test_case_id>[\w\-\.]+):\s(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)$
diff --git a/tc/iptables-basic.yaml b/tc/iptables-basic.yaml
new file mode 100644
index 0000000..ec1b25c
--- /dev/null
+++ b/tc/iptables-basic.yaml
@@ -0,0 +1,36 @@
+metadata:
+ name: iptables-basic
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: automated
+ priority: critical
+ maintainer: "Apertis Project"
+ description: "Network connection management: ensuring that iptables is available
+ and usable is part of strategic networking management
+ functionality."
+
+ macro_ostree_preconditions: iptables-basic
+
+ expected:
+ - "The automated test should succeed:"
+ - |
+ >>>> Test setup successfully!
+ >>> Running test 'test_iptables_module' ... PASS
+ >>> Running test 'test_iptables_service' ... PASS
+ >>> Running test 'test_iptables_list' ... PASS
+ >>> All tests PASSED successfully!
+
+ notes:
+ - "Make sure that you have disconnect the ethernet connection to the target
+ before you start the tethering process."
+ - "This test does not use nmap. Testing with nmap is done in another test."
+
+run:
+ steps:
+ - "# Execute the following command:"
+ - common/run-test-in-systemd --timeout=900 --name=run-test env DEBUG=2 ./run-test.sh
+
+parse:
+ pattern: ^(?P<test_case_id>[a-zA-Z0-9_\-\./]+):\s*(?P<result>pass|fail|skip|unknown)$
diff --git a/tc/tracker-indexing-local-storage.yaml b/tc/tracker-indexing-local-storage.yaml
index 1ac1a8f..ab517da 100644
--- a/tc/tracker-indexing-local-storage.yaml
+++ b/tc/tracker-indexing-local-storage.yaml
@@ -12,7 +12,7 @@ metadata:
pre-conditions:
- "Download the media tarball:"
- $ wget https://people.collabora.com/~araujo/apertis-test-media.tar.bz2
- - "Create directory and unpack media in the shared folder"
+ - "Create directory and unpack media in the shared folder."
- $ mkdir -p /home/shared/
- $ tar -jxvf apertis-test-media.tar.bz2 -C /home/shared/
diff --git a/tc/traffic-control-basic.yaml b/tc/traffic-control-basic.yaml
new file mode 100644
index 0000000..dc32acc
--- /dev/null
+++ b/tc/traffic-control-basic.yaml
@@ -0,0 +1,52 @@
+metadata:
+ name: traffic-control-basic
+ format: "Apertis Test Definition 1.0"
+ image-type: any
+ image-arch: any
+ type: functional
+ exec-type: manual
+ priority: low
+ maintainer: "Apertis Project"
+ description: "Test basic traffic control support. This test does not use tcmmd
+ but it could catch missing support in the kernel or misbehaving
+ traffic control."
+
+ resources:
+ - "A web server with a large file to download."
+
+ macro_install_packages_preconditions: wget
+
+ pre-conditions:
+ - "Check whether tcmmd >= 0.1.6 is installed:"
+ - "$ dpkg -l tcmmd"
+ - "Check that tcmmd is not running (kill it if it was launched from a
+ previous test)"
+ - "$ ps aux | grep tcmmd"
+ - "Check you have connectivity to your web server:"
+ - "$ wget http://$SERVER_IP/file"
+ - "Check what is your main network interface (eth0, ens32, wlan0...) and note
+ your IP."
+ - "$ ifconfig"
+
+ expected:
+ - "There should be no error."
+ - "Bandwidth usage with a random TCP source port should be around 4.8KB/s."
+ - "Bandwidth usage with a specific TCP source port should be around 78KB/s."
+
+ post-conditions:
+ - "If the script was interrupted, the traffic control rules can be removed
+ with:"
+ - "$ sudo tc qdisc del dev $NET_INTERFACE ingress"
+
+run:
+ steps:
+ - "As root, run the script:"
+ - "$ sudo MAIN_LINK=$NET_INTERFACE PORT=12345 /usr/share/tcmmd/tests/manual-test.sh"
+ - "The script will install traffic control rules with different ingress
+ bandwidth for TCP connections using the source port 12345 and other TCP
+ connections. The rules will wait and you will have time to check the
+ bandwidth with wget in another terminal."
+ - "Check bandwidth usage from a random TCP source port:"
+ - "$ wget http://$SERVER_IP/file -O /dev/null"
+ - "Check bandwidth usage with a specific TCP source port:"
+ - "$ wget --bind-address=$SOURCE_IP --bind-port=12345 http://$SERVER_IP/file -O /dev/null"
diff --git a/tc/x-hw-accelerated.yaml b/tc/x-hw-accelerated.yaml
index 3bc3bb4..2212c41 100644
--- a/tc/x-hw-accelerated.yaml
+++ b/tc/x-hw-accelerated.yaml
@@ -1,7 +1,7 @@
metadata:
name: x-hw-accelerated
format: "Apertis Test Definition 1.0"
- image-type: sdk
+ image-type: SDK
image-arch: amd64
type: functional
exec-type: manual
--
GitLab