diff --git a/tests-cases/apparmor-folks.yaml b/tests-cases/apparmor-folks.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1b0812fd98b54fad13b0dfb253fe369cdf55348 --- /dev/null +++ b/tests-cases/apparmor-folks.yaml @@ -0,0 +1,61 @@ +metadata: + name: apparmor-folks + format: "Apertis Test Definition 1.0" + image-type: target + image-arch: any + type: functional + exec-type: automated + priority: medium + maintainer: "Apertis Project" + description: "Test that AppArmor profiles provided in Folks package work + without unexpected auditd log entries." + + expected: + - "A list of executed test with related result (as pass/fail/skip) will be + shown on console, similar to the following:" + - | + >R1.13b_folks-alias-persistence.expected: pass + R1.13b_folks-eds-compatibility.expected: pass + R1.13b_folks-metacontacts-linking.expected: pass + R1.13b_folks-metacontacts-unlinking.expected: pass + R1.13b_folks-libsocialweb.expected: pass + R6.4.2_folks-metacontacts-antilinking.expected: pass + R1.13b_folks-retrieve-contacts-telepathy.expected: pass + test-malicious.expected: pass + + notes: + - "Other output on the console such as warnings and errors should be ignored + if the test passes" + - "This TC is based on other Folks test cases, all of them should PASS in + order to execute them against AppArmor." + - "If a subtest for this TC FAILs, check that the related test is PASSING." + - "It is worth to investigate AppArmor-side only if the original TC PASSes and + the profiled version of it FAILs." + - "Any other scenario is either OK or worth waiting the original TC to PASS." + +install: + deps: + - apparmor + - apparmor-profiles + - busybox + - chaiwala-apparmor-profiles + - apertis-tests-apparmor-report + - chaiwala-tests + - evolution-data-server + - folks-tools + - syncevolution + - python3 + - telepathy-mission-control-5 + - telepathy-gabble + - libfolks-eds25 + - libfolks-telepathy25 + +run: + steps: + - "# Execute the following commands:" + - echo -n | sudo tee /var/log/audit/audit.log + - common/run-test-in-systemd --name=run-test-folks --timeout=300 apparmor/folks/test-folks + - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING + +parse: + pattern: ^(?P<test_case_id>.+)\.expected:\s*(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)$ diff --git a/tests-cases/apparmor-tumbler.yaml b/tests-cases/apparmor-tumbler.yaml new file mode 100644 index 0000000000000000000000000000000000000000..015ea2f11a2df4dcbca77f2d7b5a5bfed7833056 --- /dev/null +++ b/tests-cases/apparmor-tumbler.yaml @@ -0,0 +1,50 @@ +metadata: + name: apparmor-tumbler + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: automated + priority: medium + maintainer: "Apertis Project" + + description: "Tests that the tumbler profile doesn't cause false negatives and + that it does not allow arbitrary reading of files in the home + directory." + + expected: + - "Both tests should report pass and not fail:" + - | + >tumbler.normal.expected: pass + tumbler.malicious.expected: pass + + notes: + - "The results of the tests printed to the standard output are enough to + determine whether they passed or fail." + - "If you get a failure and need to report a bug please run sudo watch-aa on + a terminal and run the case that is failing (separately, if both), and add + the logs you get to the bug report." + +install: + deps: + - bash + - busybox + - apertis-tests-apparmor-report + - apertis-tests-apparmor-tumbler + +run: + steps: + - "# Execute the following commands:" + - echo -n | sudo tee /var/log/audit/audit.log + - common/run-test-in-systemd --name run-test-tumbler --timeout 900 apparmor/tumbler/test-tumbler + - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING + +parse: + fixupdict: + FAILED: fail + PASSED: pass + FAIL: fail + PASS: pass + SKIP: skip + UNKNOWN: unknown + pattern: ^(?P<test_case_id>[\w\-\.]+):\s(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)$ diff --git a/tests-cases/bluez-setup.yaml b/tests-cases/bluez-setup.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48031df12e0c18b162d719df2a51493127be62c3 --- /dev/null +++ b/tests-cases/bluez-setup.yaml @@ -0,0 +1,56 @@ +metadata: + name: bluez-setup + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: manual + priority: critical + maintainer: "Apertis Project" + description: "Test setup of Bluetooth stack by running basic tests using + hciconfig. Check if the interface can be brought up/down, can be + changed of mode and can discover devices. Strategic component: + Bluetooth setup and basic functionality is part of smartphone + interaction functionality." + + resources: + - "A Bluetooth adapter." + - "A Bluetooth device." + + macro_ostree_preconditions: bluez-setup + pre-conditions: + - "Please note that connman disables bluetooth by default on a fresh image. + If it's already enabled, connmanctl will give an \"In progress\" error that + may be ignored. To enable the device:" + - $ connmanctl enable bluetooth + + expected: + - "The output should be similar to that:" + - | + >hci0: Type: BR/EDR Bus: USB + BD Address: 00:1B:DC:06:05:D4 ACL MTU: 310:10 SCO MTU: 64:8 + UP RUNNING PSCAN + RX bytes:16617 acl:0 sco:0 events:630 errors:0 + TX bytes:7102 acl:0 sco:0 commands:629 errors:8 + Features: 0xff 0xff 0x8f 0x7e 0xd8 0x1f 0x5b 0x87 + Packet type: DM1 DM3 DM5 DH1 DH3 DH5 HV1 HV2 HV3 + Link policy: RSWITCH HOLD SNIFF PARK + Link mode: SLAVE ACCEPT + Name: 'sac-target-0' + Class: 0x620100 + Service Classes: Networking, Audio, Telephony + Device Class: Computer, Uncategorized + HCI Version: 4.0 (0x6) Revision: 0x2031 + LMP Version: 4.0 (0x6) Subversion: 0x2031 + Manufacturer: Cambridge Silicon Radio (10) + + Put some devices around in discoverable mode and press ENTER + + Scanning ... + 78:47:1D:B3:6E:80 jasmin-0 + Check if the devices you enabled are in the list above + +run: + steps: + - "Execute the test suite:" + - $ ./check-setup.sh diff --git a/tests-cases/connman-pan-network-access.yaml b/tests-cases/connman-pan-network-access.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bc70f0e424c7307e2d071c3ba0bd35aedf5a5837 --- /dev/null +++ b/tests-cases/connman-pan-network-access.yaml @@ -0,0 +1,83 @@ +metadata: + name: connman-pan-network-access + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: manual + priority: medium + maintainer: "Apertis Project" + description: "Test ConnMan support for bluetooth PAN Network Access Profile. + This test will test internet access from Apertis to a bluetooth + device sharing its internet connection. The underlying connection + makes use of Bluetooth Network Access Profile with connman. First + the two devices will be paired, then connection will be + established. It will then be possible to verify that internet is + usable from Apertis." + + resources: + - "A USB Bluetooth adapter." + - "A phone supporting PAN NAP. A computer can also be used to share its + internet connection." + + macro_install_packages_preconditions: apertis-tests python3-dbus python3-gi + pre-conditions: + - "Make sure an Internet connection is active on the phone and can be shared. + The steps are phone specific." + - "Ensure the bluetooth adapter is detected in Apertis: In VirtualBox menu, + select \"Devices > USB\" and check the entry corresponding to the selected + bluetooth adapter." + - "Please note that connman disables bluetooth by default on a fresh image. + If it's already enabled, connmanctl will give an \"In progress\" error that + may be ignored. To enable the device:" + - $ connmanctl enable bluetooth + - "Start simple agent:" + - $ /usr/lib/chaiwala-tests/bluez/simple-agent -y + - "Pair both devices initiating from the phone side" + - "Pay attention to the simple-agent window as it requires input during + the pairing process." + + expected: + - "At this point, you should be able to navigate on the internet from Apertis. + For example:" + - | + >user@apertis:~$ ping www.google.fr + PING www.google.fr (216.58.198.163) 56(84) bytes of data. + 64 bytes from lhr25s10-in-f3.1e100.net (216.58.198.163): icmp_seq=1 ttl=50 time=62.0 ms + - "Start the web browser from the Applications menu and display a web page." + - "Specifying a DNS should not be necessary, it can be done as follow:" + - $ connmanctl config bluetooth_aabbccddeeff_feedbeefcafe --nameservers 8.8.8.8 + + notes: + - "Warning: Connman changes are persistent (over reboot!). After finishing + testing, it might be wise to perform the dhcp test to ensure that the + network is in a sensible state." + - "For ALL tests the enable step will give an \"In progress\" error if the + device is already enabled. This is to be expected." + + post-conditions: + - "Disconnect using connmanctl:" + - $ connmanctl disconnect bluetooth_aabbccddeeff_feedbeefcafe + - "Enable ethernet again if it was disabled" + - $ connmanctl enable ethernet + +run: + steps: + - "To ensure effective usage of bluetooth and not ethernet, one can use:" + - $ connmanctl disable ethernet + - "List connman services and Check no other network technologies is in use + with:" + - $ connmanctl services + - "From the result select the bluetooth service which matches both the adapter + and phone addresses. If connections succeeds, connmanctl will display + Connected bluetooth_aabbccddeeff_feedbeefcafe:" + - $ connmanctl connect bluetooth_aabbccddeeff_feedbeefcafe + - | + >Connected bluetooth_aabbccddeeff_feedbeefcafe + - "Control ip routing goes through PAN interface. IP addresses can vary, but + the default gateway should be bnepX or enXaabbccddeeff:" + - $ ip route + - | + >default via 192.168.2.1 dev bnep0 + 192.168.2.0/24 dev bnep0 proto kernel scope link src 192.168.2.3 + 192.168.2.1 dev bnep0 scope link diff --git a/tests-cases/folks-ofono-backend.yaml b/tests-cases/folks-ofono-backend.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcc2b1a61196c4a0c095de4d2283c361039a511c --- /dev/null +++ b/tests-cases/folks-ofono-backend.yaml @@ -0,0 +1,49 @@ +metadata: + name: folks-ofono-backend + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: manual + priority: medium + maintainer: "Apertis Project" + description: "Reading contacts from 3G modem SIM card." + + resources: + - "A 3G USB Modem." + - "A SIM card with contacts on it (SIM should *not* have a PIN configured)" + + pre-conditions: + - "Ensure Rootfs is remounted as read/write." + - $ sudo mount -o remount,rw / + - "Install dependencies" + - $ sudo apt install folks-tools apertis-tests libfolks-ofono25 connman-tests + - "Please note that connman disables cellular by default on a fresh image." + - "Make sure that the 3G USB modem is seen by Connman:" + - $ connmanctl technologies + - "should display a technology:" + - | + >/net/connman/technology/cellular + - "To enable the device (If it's already enabled, test-connman will give an + \"In progress\" error that may be ignored):" + - $ /usr/share/connman/test/test-connman enable cellular + - $ sudo reboot + + expected: + - "folks-inspect should show the personas with their details grabbed + from your SIM card." + + notes: + - "Please ignore the following warning about the eds backend: the goal of this + test is to test the ofono backend so other backends (such as the eds + backend) are explicitly disabled:" + - | + >(folks-inspect:853): folks-WARNING **: Failed to find primary PersonaStore with type ID 'eds' and ID 'system-address-book'. + Individuals will not be linked properly and creating new links between Personas will not work. + The configured primary PersonaStore's backend may not be installed. If you are unsure, check with your distribution. + +run: + steps: + - "Note: apertis-tests must be version > 20121011.0" + - "Execute test script:" + - $ /usr/lib/chaiwala-tests/contacts/folks-ofonotest.sh diff --git a/tests-cases/folks-search-contacts.yaml b/tests-cases/folks-search-contacts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b64e59eae20ed1ef592da80d71c67d59859dd8b --- /dev/null +++ b/tests-cases/folks-search-contacts.yaml @@ -0,0 +1,34 @@ +metadata: + name: folks-search-contacts + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: automated + priority: medium + maintainer: "Apertis Project" + description: "Ensures that Folks can correctly search contacts." + + expected: + - "The script will report PASSED or FAILED." + +install: + deps: + - telepathy-ring + - telepathy-mission-control-5 + - telepathy-gabble + - libfolks-eds25 + - libfolks-telepathy25 + - folks-tools + - syncevolution + +run: + steps: + - "# Execute the following command:" + - common/run-test-in-systemd --user=user --timeout=900 folks/folks-search-contacts.sh + +parse: + fixupdict: + FAILED: fail + PASSED: pass + pattern: ^(?P<test_case_id>[\w-]+):\W+(?P<result>PASSED|FAILED)$ diff --git a/tests-cases/frome.yaml b/tests-cases/frome.yaml new file mode 100644 index 0000000000000000000000000000000000000000..809f66eb02c6e4c7846bd429134a9cf7af30db8a --- /dev/null +++ b/tests-cases/frome.yaml @@ -0,0 +1,44 @@ +metadata: + name: frome + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: unit + exec-type: automated + priority: low + maintainer: "Apertis Project" + description: "Frome library test." + + macro_modules_preconditions: frome + + expected: + - "All tests should pass (ok). The output should be similar to:" + - | + >test.name0...ok + test.name1...ok + test.name2...ok + test.nameN...ok + +install: + deps: + - devscripts + - frome-tests + - gnome-desktop-testing + - python3-debian + steps: + # Download in /tmp directory + - cd /tmp/ + - apt source frome + - chown user:user -R /tmp/frome-* + +run: + steps: + - "# Execute the following commands:" + - common/run-test-in-systemd --timeout=900 --chdir /tmp/frome-* --user=user --name=frome -- sadt --verbose + +parse: + pattern: '^(?P<test_case_id>[a-zA-Z0-9_\-\./]+)\s...\s(?P<result>ok|FAIL|SKIP)' + fixupdict: + ok: pass + FAIL: fail + SKIP: skip diff --git a/tests-cases/rfkill-toggle.yaml b/tests-cases/rfkill-toggle.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b8a1510906649038c5b521f0e5955091918a341 --- /dev/null +++ b/tests-cases/rfkill-toggle.yaml @@ -0,0 +1,25 @@ +metadata: + name: rfkill-toggle + format: "Apertis Test Definition 1.0" + image-type: target + image-arch: any + type: functional + exec-type: manual + priority: medium + maintainer: "Apertis Project" + description: "Test that rfkill is functional." + + resources: + - "A bluetooth dongle is installed." + + macro_ostree_preconditions: rfkill-toggle + + expected: + - "The test script show should PASSED:" + - | + >rfkill-toggle: PASSED + +run: + steps: + - "Run the test script:" + - $ ./rfkill-toggle.sh diff --git a/tests-cases/telepathy-ring-automated.yaml b/tests-cases/telepathy-ring-automated.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e52ae6a577482656b37ccae075387afbf0a0986 --- /dev/null +++ b/tests-cases/telepathy-ring-automated.yaml @@ -0,0 +1,46 @@ +metadata: + name: telepathy-ring-automated + format: "Apertis Test Definition 1.0" + image-type: SDK + image-arch: amd64 + type: functional + exec-type: automated + priority: medium + maintainer: "Apertis Project" + description: "Support making and receiving phone calls in Telepathy. This test + is different from the manual one in that it tests with a phone + simulator, for being able to stress test the functionality + automatically. It uses a remote phone simulator instance when it + is available, otherwise, it will automatically run a local + phonesim instance and use that for the test instead." + + expected: + - "The output should be similar to this:" + - | + >telepathy-ring-automated: PASSED + +install: + deps: + - telepathy-ring + - telepathy-phoenix + - telepathy-mission-control-5 + - ofono-tests + - apertis-tests-apparmor-report + - ofono-phonesim + +run: + steps: + - "# Execute the following commands:" + - echo -n | sudo tee /var/log/audit/audit.log + - common/run-test-in-systemd --user=user sdk/telepathy-ring-automated.sh + - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING + +parse: + fixupdict: + FAILED: fail + PASSED: pass + FAIL: fail + PASS: pass + SKIP: skip + UNKNOWN: unknown + pattern: '^(?P<test_case_id>[a-zA-Z0-9_\-\./]+):\s*(?P<result>PASSED|PASS|pass|FAILED|FAIL|fail|SKIP|skip|UNKNOWN|unknown)$' diff --git a/tests-cases/traprain.yaml b/tests-cases/traprain.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35cc1a84bf0fbc4519f50c46e972fd26d3a5b1cf --- /dev/null +++ b/tests-cases/traprain.yaml @@ -0,0 +1,44 @@ +metadata: + name: traprain + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: unit + exec-type: automated + priority: medium + maintainer: "Apertis Project" + description: "Traprain library test." + + macro_modules_preconditions: traprain + + expected: + - "All tests should pass (ok). The output should be similar to:" + - | + >test.name0...ok + test.name1...ok + test.name2...ok + test.nameN...ok + +install: + deps: + - devscripts + - gnome-desktop-testing + - libtraprain-0-tests + - python3-debian + steps: + # Download in /tmp directory + - cd /tmp/ + - apt source traprain + - chown user:user -R /tmp/traprain-* + +run: + steps: + - "# Execute the following commands:" + - common/run-test-in-systemd --timeout=900 --chdir /tmp/traprain-* --user=user --name=traprain -- sadt --verbose + +parse: + pattern: '^(?P<test_case_id>[a-zA-Z0-9_\-\./]+)\s...\s(?P<result>ok|FAIL|SKIP)' + fixupdict: + ok: pass + FAIL: fail + SKIP: skip