diff --git a/tc/apparmor.yaml b/tc/apparmor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adeb7c2138a773a821f83ef9411a4c075d6834c0 --- /dev/null +++ b/tc/apparmor.yaml @@ -0,0 +1,42 @@ +metadata: + name: apparmor + format: "Apertis Test Definition 1.0" + image-type: target + image-arch: any + type: functional + exec-type: automated + priority: critical + maintainer: "Apertis Project" + description: "Security infrastructure: checking that AppArmor is available in + the kernel and active is part of strategic application and + services confinement." + + macro_ostree_preconditions: apparmor + + expected: + - "The apparmor.sh script should return 0. The return value indicates the + number of tests which failed. The ouput should be similar to the following + example:" + - | + >access.sh: PASSED - /tmp/sdtest.4283-31579-rMuWBs + capabilities.sh: FAILED - /tmp/sdtest.4548-350-UPHWk3 + - "Although extraneous lines might been shown in case of error or for debug. + Those won't interfere with the test parser." + + notes: + - "Make sure that you have disconnect the ethernet connection to the target + before you start the tethering process." + - "This testcase groups upstream developed tests mostly and therefore it's not + runnning any Apertis specific test." + +run: + steps: + - "# Execute the following command:" + - $ common/run-test-in-systemd --basename --timeout 3000 ./apparmor.sh + +parse: + fixupdict: + FAILED: fail + PASSED: pass + SKIP: skip + pattern: ^(?P<test_case_id>.+)\.sh:\s*(?P<result>PASSED|FAILED|SKIP)\s*-.* diff --git a/tc/boot-performance-automated.yaml b/tc/boot-performance-automated.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cdb18a3423e81db9d0b6ecc0030b184946ae444 --- /dev/null +++ b/tc/boot-performance-automated.yaml @@ -0,0 +1,84 @@ +metadata: + name: boot-performance-automated + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: performance + exec-type: automated + priority: high + maintainer: "Apertis Project" + description: "Test boot performance. This test checks boot performance using + systemd-analyze statistics." + + macro_ostree_preconditions: boot-performance-automated + + expected: + - "The output should be similar to this:" + - | + >Running test systemd-analyze time ... + Startup finished in 6.856s (firmware) + 436ms (loader) + 1.311s (kernel) + 2.160s (userspace) = 10.763s + (firmware) 6.856s + (loader) 436ms + (kernel) 1.311s + (userspace) 2.160s + Running test systemd-analyze blame ... + (boot.mount) 857ms + (iptables.service) 402ms + (dev-sda3.device) 380ms + (ribchester-core.service) 361ms + (ondemand.service) 266ms + (connman-wait-online.service) 262ms + (ssh.service) 150ms + (polkit.service) 130ms + (systemd-journald.service) 124ms + (systemd-logind.service) 123ms + (apertis-create-homedir.service) 117ms + (vboxvideo.service) 115ms + (connman.service) 105ms + (systemd-udev-trigger.service) 102ms + (var-rw.service) 99ms + (dev-mqueue.mount) 98ms + (systemd-remount-fs.service) 98ms + (tmp.mount) 96ms + (wpa_supplicant.service) 95ms + (dev-hugepages.mount) 95ms + (sys-kernel-debug.mount) 94ms + (apparmor.service) 81ms + (systemd-udevd.service) 64ms + (systemd-journal-flush.service) 62ms + (systemd-modules-load.service) 58ms + (systemd-tmpfiles-setup.service) 52ms + (systemd-tmpfiles-setup-dev.service) 51ms + (user@1000.service) 46ms + (kmod-static-nodes.service) 46ms + (systemd-random-seed.service) 44ms + (systemd-timesyncd.service) 43ms + (systemd-user-sessions.service) 38ms + (run-ribchester-general.mount) 34ms + (boot-efi.mount) 31ms + (etc-rw.service) 25ms + (systemd-update-utmp.service) 22ms + (run-user.mount) 21ms + (apt-daily-upgrade.service) 18ms + (systemd-sysctl.service) 18ms + (home.mount) 17ms + (media.mount) 14ms + (canterbury-init.service) 11ms + (var-Applications.mount) 10ms + (root.mount) 7ms + (etc.mount) 7ms + (apt-daily.service) 6ms + (systemd-update-utmp-runlevel.service) 5ms + (systemd-tmpfiles-clean.service) 5ms + (Applications.mount) 4ms + - "If any test failed, they will be listed instead of the success message:" + - | + >>>> The following tests FAILED: + [list of tests] + +run: + steps: + - "common/run-test-in-systemd --timeout=900 --no-lava --name=boot-performance-automated sh -c './run-test.sh | sed \"/^(.*/ s/\\$/ :pass/\"'" + +parse: + pattern: ^\((?P<test_case_id>\w[\w\d\.\@\-]+)\)\W+(?P<measurement>\d*\.?\d+)m?s\s\:(?P<result>pass) diff --git a/tc/connman.yaml b/tc/connman.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a179531355a74542faa681ae098d6dac7779dc99 --- /dev/null +++ b/tc/connman.yaml @@ -0,0 +1,49 @@ +metadata: + name: connman + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: automated + priority: critical + maintainer: "Apertis Project" + description: "Runs some automated tests using the ConnMan D-Bus API. It + guarantees most of the basic features are working. Network + connection management: ensuring that ConnMan's APIs are accessible + is part of strategic network management functionality." + + macro_ostree_preconditions: connman + + expected: + - "An example output is:" + - | + >user@apertis:~/connman-master$ bin/connman-test.sh + test_manager + test_offlinemode + test_technologies + test_technology_enable /net/connman/technology/ethernet + test_technology_tethering /net/connman/technology/ethernet + test_ethernet + test_service_properties /net/connman/service/ethernet_525400f47dad_cable + test_clock + PASSED + - FAILED is displayed in case of failure. + + notes: + - "Warning: Connman changes are persistent (over reboot!). After finishing + testing, it might be wise to perform the dhcp test to ensure that the + network is in a sensible state." + - "For ALL tests the enable step will give an \"In progress\" error if the + device is already enabled. This is to be expected." + +run: + steps: + - "# Execute the following command:" + - $ common/run-test-in-systemd --timeout=900 --basename bin/connman-test.sh + +parse: + fixupdict: + ERROR: fail + FAIL: fail + ok: pass + pattern: (?P<test_case_id>[a-z_]+) \([^\(\)]+\) \.\.\. (?P<result>ERROR|FAIL|ok) diff --git a/tc/gupnp-services.yaml b/tc/gupnp-services.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34a32aafc7244e89b79d0ff9d3a773a71881f093 --- /dev/null +++ b/tc/gupnp-services.yaml @@ -0,0 +1,23 @@ +metadata: + name: gupnp-services + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: automated + priority: medium + maintainer: "Apertis Project" + description: "Run a few tests related to service browsing, introspection, etc." + + macro_ostree_preconditions: gupnp-services + + expected: + - "PASS or FAIL for each test." + +run: + steps: + - "# Execute the following command:" + - common/run-test-in-systemd --basename --timeout=900 ./run-test.sh + +parse: + pattern: ^(?P<test_case_id>[a-zA-Z0-9_\-\./]+):\s*(?P<result>pass|fail|skip|unknown)$ diff --git a/tc/tracker-indexing-mass-storage.yaml b/tc/tracker-indexing-mass-storage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ce71c5478fb58376db67a735818f1b644a2afe7 --- /dev/null +++ b/tc/tracker-indexing-mass-storage.yaml @@ -0,0 +1,75 @@ +metadata: + name: tracker-indexing-mass-storage + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: manual + priority: medium + maintainer: "Apertis Project" + description: "Check tracker indexes mass storage devices on insertion." + resources: + - "A USB mass storage device." + - "Format mass storage device with a single FAT32 partition." + + macro_install_packages_preconditions: apertis-tests gvfs libglib2.0-bin python3 + python3-gi gir1.2-tracker-1.0 tracker + + expected: + - "The output should be similar to this:" + - | + >Please insert storage ... + Warning: Stopping tracker-store.service, but it can still be activated by: + tracker-store.timer + Warning: Stopping tracker-miner-fs.service, but it can still be activated by: + tracker-miner-fs.timer + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Userguides' initial status is 'Idle' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Applications' initial status is 'Idle' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' initial status is 'Crawl finished for directory 'file:///home/shared'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Extract' initial status is 'Idle' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Userguides' status changed to 'Initializing' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Userguides' status changed to 'Idle' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Applications' status changed to 'Crawling recursively directory 'file:///usr/local/share/applications'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Applications' status changed to 'Crawling recursively directory 'file:///usr/share/applications'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Applications' status changed to 'Crawl finished for directory 'file:///home/user/.local/share/desktop-directories'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' status changed to 'Crawling recursively directory 'file:///media/user/6114-D5FD'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Applications' status changed to 'Idle' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' status changed to 'Crawling recursively directory 'file:///home/user/Documents'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' status changed to 'Crawling recursively directory 'file:///home/user/Downloads'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' status changed to 'Crawling recursively directory 'file:///home/user/Music'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' status changed to 'Crawling recursively directory 'file:///home/user/Pictures'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' status changed to 'Crawling recursively directory 'file:///home/user/Videos'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' status changed to 'Crawling recursively directory 'file:///home/shared'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' status changed to 'Crawl finished for directory 'file:///home/shared'' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' status changed to 'Processing…' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Files' status changed to 'Idle' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Extract' status changed to 'Extracting metadata' + TrackerIndexer: Miner 'org.freedesktop.Tracker1.Miner.Extract' status changed to 'Idle' + TrackerIndexer: quit main loop + . + ---------------------------------------------------------------------- + Ran 1 test in 16.406s + + OK + - "If any test failed, they will be listed instead of the success message:" + - | + >====================================================================== + FAIL: test_removable_device (__main__.TestRemovableDevice) + ---------------------------------------------------------------------- + Traceback (most recent call last): + File "./apertis-tests/tracker/manual/test-removable-device.py", line 46, in test_removable_device + self.assertTrue(False) + AssertionError: False is not true + + ---------------------------------------------------------------------- + Ran 1 test in 13.306s + + FAILED (failures=1) + +run: + steps: + - "Execute the test script:" + - $ /usr/lib/apertis-tests/tracker/manual/test-removable-device.py + - "Insert the USB mass storage device when prompted to do so. You may need to + click on it in the file manager to mount it so the test can proceed." + - "The test will continue once the device is automounted." diff --git a/tc/webkit2gtk-aligned-scroll.yaml b/tc/webkit2gtk-aligned-scroll.yaml new file mode 100644 index 0000000000000000000000000000000000000000..382715fb5bf83827484f9cda48d3d510a676a31f --- /dev/null +++ b/tc/webkit2gtk-aligned-scroll.yaml @@ -0,0 +1,40 @@ +metadata: + name: webkit2gtk-aligned-scroll + format: "Apertis Test Definition 1.0" + image-type: target + image-arch: any + type: functional + exec-type: manual + priority: low + maintainer: "Apertis Project" + description: "Test that scrolling is pinned in a given direction when started + mostly towards it." + resources: + - "A touch-screen and a mouse (test with both)." + + macro_install_packages_preconditions: webkit2gtk-testing + + expected: + - "When the scroll is started by a diagonal drag (1), you should be able to + pan the page freely." + - "When the scroll is started by a vertical drag (2), you should only be able + to pan the page vertically, regardless of if you move your finger/mouse + horizontally." + - "When the scroll is started by a horizontal drag (3), you should only be + able to pan the page horizontally, regardless of if you move your + finger/mouse vertically." + + notes: + - "Check the following example video for the expected result: + https://wiki.apertis.org/static/Aligned-scroll.ogv" + - "Both mouse and touchscreen need to PASS for this test case to be considered + a PASS. If either does not pass, then the test case has failed." + +run: + steps: + - "Run the test program:" + - $ GtkClutterLauncher -g 400x600 http://gnome.org/ + - "Try scrolling by starting a drag diagonally (1)" + - "Try scrolling by starting a drag vertically (2)" + - "Try scrolling by starting a drag horizontally, ensure you can only pan the + page horizontally (3)" diff --git a/tc/webkit2gtk-contextual-zoom.yaml b/tc/webkit2gtk-contextual-zoom.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e0106cc396aeb2e4f11ffb490da37462b800e93 --- /dev/null +++ b/tc/webkit2gtk-contextual-zoom.yaml @@ -0,0 +1,38 @@ +metadata: + name: webkit2gtk-contextual-zoom + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: manual + priority: low + maintainer: "Apertis Project" + description: "This test case is about zooming parts of a web page to make it + easier to read on devices with limited screen area. The scope of + the test is the zooming functionality that takes into account the + content in the clicked area, other functionality that belongs to + a real browser isn't to be tested here." + + resources: + - "This test case requires the use of a mouse, touch gestures aren't supported + as of now." + + macro_install_packages_preconditions: webkit2gtk-testing + + expected: + - "The clicked paragraphs should be focused as shown in the following video: + https://www.apertis.org/static/Contextual-zoom.ogv" + + notes: + - "This test changes the zoom factor of the WebView, which is not reset on new + pages loaded, so close GtkClutterLauncher before running other tests." + +run: + steps: + - "Launch the test application with each URL in resources:" + - $ GtkClutterLauncher file:///usr/share/webkit2gtk/testing/LayoutTests/fast/text-autosizing/clusters-sufficient-width.html + - "Double click on the paragraph to the right." + - "Double click anywhere to get out of contextual zoom." + - "Double click the second paragraph to the left." + - "Double click anywhere to get out of contextual zoom." + - "Double click other paragraphs at will." diff --git a/tc/webkit2gtk-drag-and-drop.yaml b/tc/webkit2gtk-drag-and-drop.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fce183fc1378277aa5439e96a6b6e763dd6f8225 --- /dev/null +++ b/tc/webkit2gtk-drag-and-drop.yaml @@ -0,0 +1,47 @@ +metadata: + name: webkit2gtk-drag-and-drop + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: manual + priority: low + maintainer: "Apertis Project" + description: "Test HTML5 drag-and-drop support in WebKit2GTK." + + resources: + - "A touch-screen." + - "A mouse." + + macro_install_packages_preconditions: webkit2gtk-testing + + expected: + - "Dragging with the mouse the first \"Drag me!\" box to the \"Drop here\" box + below should copy the dragged item and should not pan the page." + - "Dragging with a single touch the first \"Drag me!\" box to the \"Drop + here\" box below should copy the dragged item and should not pan the page." + - "Dragging with the mouse any \"Drag me!\" box from the main page to the + \"Drop here\" box in the subframe below should copy the dragged item and + should not pan the page." + - "Dragging with a single touch any \"Drag me!\" box from the main page to the + \"Drop here\" box in the subframe below should copy the dragged item and + should not pan the page" + - "Dragging with the mouse any \"Drag me!\" box from the subframe to the \"Drop + here\" box in the main page below should copy the dragged item and should + not pan the page." + - "Dragging with a single touch any \"Drag me!\" box from the subframe to the + \"Drop here\" box in the main page below should copy the dragged item and + should not pan the page." + - "Dragging with the mouse the big images under the \"Drop here\" boxes should + not initiate dragging but should pan the page." + - "Dragging with a single touch the big images under the \"Drop here\" boxes + should not initiate dragging but should pan the page." + - "Dragging with the mouse in any point outside the \"Drag me!\" boxes should + pan the page." + - "Dragging with a single touch in any point outside the \"Drag me!\" boxes + should pan the page." + +run: + steps: + - "Run the test program:" + - $ GtkClutterLauncher file:///usr/share/webkit2gtk/testing/drag.html