Add command line interface to the renderer

- Adds a command line interface to the renderer to process a single test case yaml file or a directory of files. - Adds the feature to write the test case html page inside a specific directory. - Port few more test cases to the yaml format. Signed-off-by: Luis Araujo <luis.araujo@collabora.co.uk>

Add command line interface to the renderer
- Adds a command line interface to the renderer to process a single test case yaml file or a directory of files. - Adds the feature to write the test case html page inside a specific directory. - Port few more test cases to the yaml format. Signed-off-by: Luis Araujo <luis.araujo@collabora.co.uk>
c78e403e · Luis Araujo · 1c7dcb80 · c78e403e · c78e403e · c78e403e
Commit c78e403e authored 6 years ago by Luis Araujo
--- a/renderer/make_page.py
+++ b/renderer/make_page.py
@@ -3,9 +3,12 @@
 # Make a web page from a YAML test case file.
 #

+import os
 import sys
 import yaml
+import shutil
 from jinja2 import Environment, FileSystemLoader
+from argparse import ArgumentParser

 from parser import parse_format, test_case_format

@@ -130,21 +133,57 @@ def get_template_values(testcase_data):

    return template_values

-
-if '__main__' == __name__:
-    testcase_file = sys.argv[1]
-
+def generate_test_case(tc_file, directory):
    try:
-        with open(testcase_file) as testcase:
+        with open(tc_file) as testcase:
            tc_data = yaml.safe_load(testcase)
    except yaml.scanner.ScannerError as e:
        print("yaml format error:", e)
        sys.exit(1)

+    # Parse file to detect any syntax error.
+    print("Parsing file", tc_file)
    parse_format(tc_data, test_case_format)

    env = Environment(loader=FileSystemLoader([TEMPLATE_DIR]))
    # Get template from environment and render it.
-    data = env.get_template('index.html').render(get_template_values(tc_data))
+    data = env.get_template('templates/index.html').render(get_template_values(tc_data))
+
+    filename = os.path.splitext(os.path.basename(tc_file))[0] + ".html"
+    print("Generating test case page", filename)
+    with open(os.path.join(directory, filename), 'w') as html_file:
+        html_file.write(data)
+

-    print(data)
+if '__main__' == __name__:
+    cli_parser = ArgumentParser(description="make_page")
+    cli_parser.add_argument('-d', '--test-case-dir',
+                            help="Directory path for generated test cases")
+    cli_parser.add_argument('yaml_files',
+                            help="YAML file or files directory")
+    args = cli_parser.parse_args()
+
+    directory = '.'
+    if args.test_case_dir:
+        directory = args.test_case_dir
+        try:
+            os.mkdir(directory)
+        except FileExistsError:
+            print("Directory '{}' already exists".format(directory))
+        except e:
+            print("Error:", e)
+            sys.exit(1)
+
+    if os.path.isfile(args.yaml_files):
+        generate_test_case(args.yaml_files, directory)
+    else:
+        for root, _, files in os.walk(args.yaml_files):
+            for f in files:
+                tc_file = os.path.join(root, f)
+                if os.path.isfile(tc_file):
+                    generate_test_case(tc_file, directory)
+
+    cssdir = os.path.join(directory, 'css')
+    if not (os.path.exists(cssdir) and os.path.isdir(cssdir)):
+        print("Copying css style to", cssdir)
+        shutil.copytree('css/', cssdir)
--- a/renderer/index.html
+++ b/renderer/index.html
@@ -7,7 +7,7 @@
    <title>{{ name }}</title>
  </head>
  <body>
-    {% import 'macros.html' as macros %}
+    {% import 'templates/macros.html' as macros %}
    <main role="main" class="container" style="margin-top: 40px; margin-bottom: 40px">
      <h2>{{ name }} <small class="text-muted">{{ exec_type }}</small></h2>
      <h3><span class="badge badge-{{ priority_color }}">{{ priority }}</span></h3>

--- a/renderer/macros.html
+++ b/renderer/macros.html
--- a/tc/apparmor-tracker.yaml
+++ b/tc/apparmor-tracker.yaml
+metadata:
+  name: apparmor-tracker
+  format: "Apertis Test Definition 1.0"
+  image-type: any
+  image-arch: any
+  type: functional
+  exec-type: automated
+  priority: medium
+  maintainer: "Apertis Project"
+  description: "Tests that the tracker abstraction and tracker profiles don't
+                cause false negatives and that it does not allow arbitrary reading
+                of files in the home directory."
+
+  pre-conditions:
+    - "Download the media tarball:"
+    - $ wget https://people.collabora.com/~araujo/apertis-test-media.tar.bz2
+    - "Create directory and unpack media in the shared folder."
+    - $ mkdir -p /home/shared/
+    - $ tar -jxvf apertis-test-media.tar.bz2 -C /home/shared/
+
+  expected:
+    - "Both tests should report pass and not fail:"
+    - |
+        >tracker.normal.expected: pass
+        tracker.malicious.expected: pass
+
+  notes:
+    - "Make sure that you have disconnect the ethernet connection to the target
+       before you start the tethering process."
+    - "The results of the tests printed to the standard output are enough to
+       determine whether they passed or fail."
+    - "If you get a failure and need to report a bug please run sudo watch-aa on
+       a terminal and run the case that is failing (separately, if both), and add
+       the logs you get to the bug report."
+
+install:
+  deps:
+  - busybox
+  - apertis-tests-apparmor-report
+  - apertis-tests-apparmor-tracker
+  - gir1.2-grilo-0.2
+  - gir1.2-tracker-1.0
+  - python3
+  - python3-gi
+  - wget
+  steps:
+  - wget https://people.collabora.com/~araujo/apertis-test-media.tar.bz2
+  - mkdir -p /home/shared/
+  - tar -jxvf apertis-test-media.tar.bz2 -C /home/shared/
+
+run:
+  steps:
+    - "# Execute the following commands:"
+    - echo -n | sudo tee /var/log/audit/audit.log
+    - common/run-test-in-systemd --name run-test-tracker --timeout 3000 apparmor/tracker/test-tracker
+    - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
+
+parse:
+  pattern: ^(?P<test_case_id>[\w\-\.]+):\s(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)$
--- a/tc/iptables-basic.yaml
+++ b/tc/iptables-basic.yaml
+metadata:
+  name: iptables-basic
+  format: "Apertis Test Definition 1.0"
+  image-type: any
+  image-arch: any
+  type: functional
+  exec-type: automated
+  priority: critical
+  maintainer: "Apertis Project"
+  description: "Network connection management: ensuring that iptables is available
+                and usable is part of strategic networking management
+                functionality."
+
+  macro_ostree_preconditions: iptables-basic
+
+  expected:
+    - "The automated test should succeed:"
+    - |
+        >>>> Test setup successfully!
+        >>> Running test 'test_iptables_module' ... PASS
+        >>> Running test 'test_iptables_service' ... PASS
+        >>> Running test 'test_iptables_list' ... PASS
+        >>> All tests PASSED successfully!
+
+  notes:
+    - "Make sure that you have disconnect the ethernet connection to the target
+       before you start the tethering process."
+    - "This test does not use nmap. Testing with nmap is done in another test."
+
+run:
+  steps:
+    - "# Execute the following command:"
+    - common/run-test-in-systemd --timeout=900 --name=run-test env DEBUG=2 ./run-test.sh
+
+parse:
+  pattern: ^(?P<test_case_id>[a-zA-Z0-9_\-\./]+):\s*(?P<result>pass|fail|skip|unknown)$
--- a/tc/tracker-indexing-local-storage.yaml
+++ b/tc/tracker-indexing-local-storage.yaml
@@ -12,7 +12,7 @@ metadata:
  pre-conditions:
    - "Download the media tarball:"
    - $ wget https://people.collabora.com/~araujo/apertis-test-media.tar.bz2
-    - "Create directory and unpack media in the shared folder"
+    - "Create directory and unpack media in the shared folder."
    - $ mkdir -p /home/shared/
    - $ tar -jxvf apertis-test-media.tar.bz2 -C /home/shared/


--- a/tc/traffic-control-basic.yaml
+++ b/tc/traffic-control-basic.yaml
+metadata:
+  name: traffic-control-basic
+  format: "Apertis Test Definition 1.0"
+  image-type: any
+  image-arch: any
+  type: functional
+  exec-type: manual
+  priority: low
+  maintainer: "Apertis Project"
+  description: "Test basic traffic control support. This test does not use tcmmd
+                but it could catch missing support in the kernel or misbehaving
+                traffic control."
+
+  resources:
+    - "A web server with a large file to download."
+
+  macro_install_packages_preconditions: wget
+
+  pre-conditions:
+    - "Check whether tcmmd >= 0.1.6 is installed:"
+    - "$ dpkg -l tcmmd"
+    - "Check that tcmmd is not running (kill it if it was launched from a
+       previous test)"
+    - "$ ps aux | grep tcmmd"
+    - "Check you have connectivity to your web server:"
+    - "$ wget http://$SERVER_IP/file"
+    - "Check what is your main network interface (eth0, ens32, wlan0...) and note
+       your IP."
+    - "$ ifconfig"
+
+  expected:
+    - "There should be no error."
+    - "Bandwidth usage with a random TCP source port should be around 4.8KB/s."
+    - "Bandwidth usage with a specific TCP source port should be around 78KB/s."
+
+  post-conditions:
+    - "If the script was interrupted, the traffic control rules can be removed
+       with:"
+    - "$ sudo tc qdisc del dev $NET_INTERFACE ingress"
+
+run:
+  steps:
+    - "As root, run the script:"
+    - "$ sudo MAIN_LINK=$NET_INTERFACE PORT=12345 /usr/share/tcmmd/tests/manual-test.sh"
+    - "The script will install traffic control rules with different ingress
+       bandwidth for TCP connections using the source port 12345 and other TCP
+       connections. The rules will wait and you will have time to check the
+       bandwidth with wget in another terminal."
+    - "Check bandwidth usage from a random TCP source port:"
+    - "$ wget http://$SERVER_IP/file -O /dev/null"
+    - "Check bandwidth usage with a specific TCP source port:"
+    - "$ wget --bind-address=$SOURCE_IP --bind-port=12345 http://$SERVER_IP/file -O /dev/null"
--- a/tc/x-hw-accelerated.yaml
+++ b/tc/x-hw-accelerated.yaml
 metadata:
  name: x-hw-accelerated
  format: "Apertis Test Definition 1.0"
-  image-type: sdk
+  image-type: SDK
  image-arch: amd64
  type: functional
  exec-type: manual