.gitignore

@@ -23,12 +23,11 @@ apparmor/webkit-clutter/webkit-clutter
 /sdk/automated/multipath.c.gcov
 /sdk/automated/multipath.gcda
 /sdk/automated/multipath.gcno
-/sdk/automated/sysprof-out.xml
+/sdk/automated/sysprof-out.capture
 # Misc compiled files
 /clutter-zoom/clutter-zoom-test
 /emulated-soft-keys/keyevent_waiter
 /contacts/folks-extended-info
-/contacts/folks-sync-only
 
 # gettext .gmo files
 *.gmo

Makefile

@@ -4,53 +4,17 @@ pkglibdir = /usr/lib/apertis-tests
 
 SUBDIRS = \
 	apparmor/goals \
-	apparmor/dbus \
-	apparmor/folks \
 	apparmor/geoclue \
-	apparmor/gstreamer1.0 \
-	apparmor/libreoffice \
 	apparmor/tracker \
-	apparmor/tumbler \
-	apparmor/webkit-clutter \
 	apparmor/ofono \
-	apparmor/pulseaudio \
-	apparmor/session-lockdown \
-	apparmor/automated/ \
 	bluez/ \
 	boot-performance/automated/ \
-	cgroups/automated/ \
-	clutter-bigger-reactive-area/ \
-	clutter-i18n/manual/ \
-	clutter-performance/ \
-	clutter-rotate/ \
-	clutter-transparent-stage/ \
-	clutter-zoom/ \
 	common/ \
 	dbus/ \
-	emulated-soft-keys/ \
-	firewall/automated/ \
-	geoclue/automated/ \
-	gettext/automated/ \
-	glib/automated/ \
-	gstreamer-buffering/automated/ \
-	gupnp/automated/ \
-	libreoffice/automated/ \
-	librest/automated/ \
-	libsoup/automated/ \
 	networking/proxy-manual/ \
-	resources/ \
-	sqlite/automated/ \
 	traffic-control/manual/ \
-	tumbler/automated/ \
-	v4l/automated/ \
 	sdk/automated/ \
 	connman \
-	contacts \
-	gstreamer-decode \
-	gstreamer-viv-direct-texture/automated/ \
-	x-out-of-screen-events \
-	telepathy \
-	rfkill \
 	$(NULL)
 
 # Bits that are architecture-independent and can just be copied
@@ -58,20 +22,17 @@ COPY = \
 	$(wildcard apparmor/*.sh) \
 	$(wildcard apparmor/*.yaml) \
 	apertis_tests_lib \
-	apparmor/automated \
-	apparmor/folks \
 	apparmor/run-aa-test \
-	clutter \
 	common \
+        connman \
 	dbus \
 	folks \
 	grilo \
+	gstreamer-decode \
 	inherit-config.sh \
 	misc \
-	predeployed-misc \
 	resources \
 	sdk \
-	templates \
 	tracker \
 	$(NULL)
 

README

-= Templates for Apertis tests =
-
-The templates contains the set of templates we use for lava testing for Apertis
-and a set of default profiles. These are intended to be used with the lqa tool,
-available from: http://cgit.collabora.com/git/singularity/tools/lqa.git/
-
-For lqa usage see the readme in the lqa source tree, minimal examples
-for Apertis below:
-
-To submit e.g. a minimal boot test to minnowboard (daily 16.03) targets use:
-
-    lqa submit -g templates/profiles.yaml \
-      -p apertis-16.03-daily-minnowboard-max-target \
-      -t image_date:20160229.0 \
-      templates/minimal-tpl.json
-
-The image_date variable specifies the image data of the image to use, see
-https://images.apertis.org/daily/16.03/ for the list of
-available images.
-
-Note that lqa can replace any variable (either json items or {{variable}}) this
-can for example be used to change the git-repo value to non-standard repository
-to test tests before merging them for example:
-
-    lqa submit -g templates/profiles.yaml \
-      -p apertis-16.03-daily-minnowboard-max-target \
-      -t image_date:20160229.0 \
-      -t git-repo:https://...@git.apertis.org/git/users/me/apertis-tests.git \
-      -t revision:test \
-      templates/minimal-tpl.json
-
-
-For developer testing, some machines have a predeployed image available, saving
-the time to re-deploy images on every test boot. An example minimal template
-for that is available in templates/minimal-predeployed-1503-target-tpl.json (at
-the time of this writing only setup for minnowboard max). Note that this images
-are fully ephemeral, all changes to the filesystem will disappear on each boot

README.md

+## Templates for Apertis tests
+
+**Warning: The content of this repository are being moved to https://gitlab.apertis.org/tests/
+to improve the testing worflow. New tests should be added to the new location.**
+
+The templates contains the set of templates we use for lava testing for Apertis
+and a set of default profiles. These are intended to be used with the lqa tool,
+available from: http://cgit.collabora.com/git/singularity/tools/lqa.git/
+
+For lqa usage see the readme in the lqa source tree, minimal examples
+for Apertis below:
+
+To submit the testsuite for headless images for minnowboard targets use:
+
+    lqa submit -g ./apertis-tests/templates/profiles.yaml \
+      --profile apertis-minimal-amd64-uefi \
+      -t release:v2021dev1 \
+      -t image_date:20191230.0 \
+      -t image_name:apertis_v2021dev1-minimal-amd64-uefi_20191230.0
+
+If you just want to issue a quicker boot test which does not execute the
+testcases, specify the `minimal-tpl.yaml` job:
+
+    lqa submit -g ./apertis-tests/templates/profiles.yaml \
+      --profile apertis-minimal-amd64-uefi \
+      -t release:v2021dev1 \
+      -t image_date:20191230.0 \
+      -t image_name:apertis_v2021dev1-minimal-amd64-uefi_20191230.0 \
+      ./apertis-tests/templates/minimal-tpl.yaml
+
+The image_date variable specifies the image data of the image to use, see
+https://images.apertis.org/daily/ for the list of available images.

ade/ade-tests-custom.sh

+#!/bin/sh
+#
+# Copyright © 2016 Collabora Ltd.
+#
+# SPDX-License-Identifier: MPL-2.0
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# ade-tests-custom.sh
+#
+# This test script will exercise some ade features in order to catch issues in what real users may try out:
+# * listing installed sysroots
+# * listing available sysroots
+# * downloading sysroots
+# * verifying sysroots
+# * installing sysroots
+# * targeting different distributions, releases and architectures
+# * cross-building
+# * exporting app-bundles
+# * default and non-default parameters
+#
+# It needs:
+# * network access to fetch sysroots for arm64 and armhf
+# * network access to clone the sample source repository
+# * the arm64 gcc-aarch64-linux-gnu cross-toolchain
+# * the armhf gcc-arm-linux-gnueabi cross-toolchain
+
+set -eux
+
+. /etc/os-release # import $ID and $VERSION_ID
+
+LANG=C.UTF-8
+DISTRIBUTION=$ID
+RELEASE=$VERSION_ID
+ARCH=arm64 # the non-default one to be tested
+SYSROOTINDEX="https://images.apertis.org/sysroot/$RELEASE/sysroot-$DISTRIBUTION-$RELEASE-$ARCH"
+SAMPLEREPO=https://gitlab.apertis.org/pkg/dash/
+
+git clone $SAMPLEREPO sample -b "apertis/$RELEASE"
+
+ade sysroot list
+
+echo ==== Testing ade usage with custom parameters
+
+ade sysroot installed --distro $DISTRIBUTION --release $RELEASE --arch $ARCH
+
+ade sysroot latest --distro $DISTRIBUTION --release $RELEASE --arch $ARCH
+
+ade sysroot latest --distro $DISTRIBUTION --release $RELEASE --arch $ARCH --url "$SYSROOTINDEX"
+
+ade --format parseable sysroot download --distro $DISTRIBUTION --release $RELEASE --arch $ARCH --url "$SYSROOTINDEX" --dest /opt/sysroot/downloads/
+
+ade sysroot verify --file /opt/sysroot/downloads/sysroot-$DISTRIBUTION-$RELEASE-${ARCH}_*.tar.gz
+
+ade --format parseable sysroot install --file /opt/sysroot/downloads/sysroot-$DISTRIBUTION-$RELEASE-${ARCH}_*.tar.gz
+
+ade sysroot list
+
+cd sample
+
+ade configure --debug --sysroot $DISTRIBUTION-$RELEASE-$ARCH -- --disable-Werror
+ade build
+
+stat src/dash
+
+test "$(stat -c %s src/dash)" -gt 0
+
+rm -r ../sample

ade/ade-tests-defaults.sh

+#!/bin/sh
+#
+# Copyright © 2016 Collabora Ltd.
+#
+# SPDX-License-Identifier: MPL-2.0
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+# ade-tests-defaults.sh
+#
+# This test script will exercise some ade features in order to catch issues in what real users may try out:
+# * listing installed sysroots
+# * listing available sysroots
+# * downloading sysroots
+# * verifying sysroots
+# * installing sysroots
+# * targeting different distributions, releases and architectures
+# * cross-building
+# * exporting app-bundles
+# * default and non-default parameters
+#
+# It needs:
+# * network access to fetch sysroots for arm64 and armhf
+# * network access to clone the sample source repository
+# * the arm64 gcc-aarch64-linux-gnu cross-toolchain
+# * the armhf gcc-arm-linux-gnueabi cross-toolchain
+
+set -eux
+
+. /etc/os-release # import $ID and $VERSION_ID
+
+LANG=C.UTF-8
+DISTRIBUTION=$ID
+RELEASE=$VERSION_ID
+SAMPLEREPO=https://gitlab.apertis.org/pkg/dash/
+
+git clone $SAMPLEREPO sample -b "apertis/$RELEASE"
+
+ade sysroot list
+
+echo ==== Testing ade usage with default settings
+
+ade sysroot installed
+
+ade sysroot latest
+
+ade --format parseable sysroot install
+
+ade sysroot list
+
+cd sample
+
+ade configure --debug --sysroot $DISTRIBUTION-$RELEASE-armhf
+ade build
+
+stat src/dash
+
+test "$(stat -c %s src/dash)" -gt 0
+
+rm -r ../sample

apertis_tests_lib/bluez.py

@@ -554,7 +554,7 @@ class AdapterDeviceTester():
                i['org.bluez.Device1']['Adapter'] == \
                self.__adapter_obj.object_path:
                 address = str(i['org.bluez.Device1']['Address'])
-                name = str(i['org.bluez.Device1'].get(['Name'], address))
+                name = str(i['org.bluez.Device1'].get('Name', address))
 
                 print('%s: %s, %s' % (p, address, name))
         device_path = input("Select one device to pair with: ")

apertis_tests_lib/grilo.py

@@ -42,7 +42,7 @@ class GriloBrowserMixin:
             medias = self.source.browse_sync(media, keys, options)
 
         for media in medias:
-            if isinstance(media, Grl.MediaBox):
+            if media.is_container():
                 self.__browse_recurse(media)
                 continue
 

apertis_tests_lib/tracker.py

@@ -27,7 +27,7 @@ class TrackerIndexerMixin:
             shell=True)
 
         print("TrackerIndexerMixin: Reset tracker's DB")
-        subprocess.check_call('tracker reset -r',
+        subprocess.check_call('yes | tracker reset -r',
                               stdout=subprocess.DEVNULL,
                               shell=True)
 

apparmor/apparmor-basic-profiles.yaml

-metadata:
-  name: apparmor-basic-profiles
-  format: "Lava-Test-Shell Test Definition 1.0"
-  description: "Test basic apparmor profiles. Only parsing and 
-                syntax of profiles are tested for now."
-  maintainer: "luis.araujo@collabora.co.uk"
-  scope:
-  - functional
-  devices:
-  - i386
-  environment:
-  - lava-test-shell
-
-run:
-  steps:
-    - common/run-test-in-systemd --user=user --timeout=90 --basename apparmor/automated/run-test.sh --debug
-
-parse:
-  pattern: ^(?P<test_case_id>[a-zA-Z0-9_\-\./]+):\s*(?P<result>pass|fail|skip|unknown)$

apparmor/apparmor-chaiwala-system.yaml

-metadata:
-  name: apparmor-chaiwala-system
-  format: "Lava-Test-Shell Test Definition 1.0"
-  description: "Run chaiwala-apparmor test suite"
-  maintainer: "luis.araujo@collabora.co.uk"
-  scope:
-  - functional
-  devices:
-  - i386
-  environment:
-  - lava-test-shell
-
-install:
-  deps:
-  - busybox
-  - apertis-tests-apparmor-goals
-  - apertis-tests-apparmor-report
-
-run:
-  steps:
-  - echo -n | sudo tee /var/log/audit/audit.log
-  - common/run-test-in-systemd --timeout=90 --basename /usr/lib/apertis-tests/apparmor/goals/lava_run_tests
-  - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
-
-parse:
-  fixupdict:
-    FAILED: fail
-    PASSED: pass
-    SKIP: skip
-  pattern: ^(?P<test_case_id>.+)\:\s(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)\s-
-

apparmor/apparmor-dbus.yaml

-metadata:
-  name: apparmor-dbus
-  format: "Lava-Test-Shell Test Definition 1.0"
-  description: "Tests that the D-Bus mediation in AppArmor works properly."
-  maintainer: "luis.araujo@collabora.co.uk"
-  scope:
-  - functional
-  devices:
-  - i386
-  environment:
-  - lava-test-shell
-
-install:
-  deps:
-  - busybox
-  - apertis-tests-apparmor-dbus
-  - apertis-tests-apparmor-report
-
-run:
-  steps:
-  - echo -n | sudo tee /var/log/audit/audit.log
-  - "common/run-test-in-systemd --name=load-profiles --timeout=90 apparmor_parser --reload --skip-cache /etc/apparmor.d/usr.lib.apertis-tests.apparmor.dbus.dbus-*.py"
-  - "common/run-test-in-systemd --name=run-test-dbus --timeout=90 /usr/lib/apertis-tests/apparmor/dbus/test-dbus"
-  - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
-
-parse:
-  pattern: ^(?P<test_case_id>[\w\-\.]+):\s(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)$
-

apparmor/apparmor-folks.yaml

-metadata:
-  name: apparmor-folks
-  format: "Lava-Test-Shell Test Definition 1.0"
-  description: "Run apparmor-folks test suite"
-  maintainer: "luis.araujo@collabora.co.uk"
-  scope:
-  - functional
-  devices:
-  - i386
-  environment:
-  - lava-test-shell
-
-install:
-  deps:
-  - apparmor
-  - apparmor-profiles
-  - busybox
-  - chaiwala-apparmor-profiles
-  - apertis-tests-apparmor-report
-  - chaiwala-tests
-  - evolution-data-server
-  - folks-tools
-  - syncevolution
-  - python3
-  - telepathy-mission-control-5
-  - telepathy-gabble
-  - libfolks-eds25
-  - libfolks-telepathy25
-
-run:
-  steps:
-  - echo -n | sudo tee /var/log/audit/audit.log
-  - common/run-test-in-systemd --name=run-test-folks --timeout=300 apparmor/folks/test-folks
-  - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
-
-parse:
-  pattern: ^(?P<test_case_id>.+)\.expected:\s*(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)$
-

apparmor/apparmor-functional-demo.yaml

-metadata:
-  name: apparmor-functional-demo
-  format: "Lava-Test-Shell Test Definition 1.0"
-  description: "Run apparmor-demo test suite"
-  maintainer: "luis.araujo@collabora.co.uk"
-  scope:
-  - functional
-  devices:
-  - i386
-  environment:
-  - lava-test-shell
-
-install:
-  deps:
-  - apparmor-demo
-  - busybox
-  - apertis-tests-apparmor-report
-
-run:
-  steps:
-  - echo -n | sudo tee /var/log/audit/audit.log
-  - common/run-test-in-systemd --name=run-lava_run_tests --timeout=90 /usr/lib/apparmor-demo/lava_run_tests
-  - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
-
-parse:
-  fixupdict:
-    FAILED: fail
-    PASSED: pass
-    SKIP: skip
-  pattern: ^(?P<test_case_id>.+)\.sh:\s*(?P<result>PASS|PASSED|pass|FAIL|FAILED|fail|SKIP|skip|UNKNOWN|unknown)\s*-
-

apparmor/apparmor-geoclue.yaml

-metadata:
-  name: apparmor-geoclue
-  format: "Lava-Test-Shell Test Definition 1.0"
-  description: "Tests that the geoclue profile works properly does not allow arbitrary 
-                reading of files in the home directory."
-  maintainer: "luis.araujo@collabora.co.uk"
-  scope:
-  - functional
-  devices:
-  - i386
-  environment:
-  - lava-test-shell
-
-install:
-  deps:
-  - busybox
-  - apertis-tests-apparmor-geoclue
-
-run:
-  steps:
-  - echo -n | sudo tee /var/log/audit/audit.log
-  - common/run-test-in-systemd --name run-test-geoclue --timeout 90 /usr/lib/apertis-tests/apparmor/geoclue/test-geoclue
-  - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
-
-parse:
-  pattern: ^(?P<test_case_id>[\w\-\.]+):\s(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)$
-

apparmor/apparmor-gstreamer1-0.yaml

-metadata:
-  name: apparmor-gstreamer1-0
-  format: "Lava-Test-Shell Test Definition 1.0"
-  description: "Run apparmor-gstreamer1.0 test suite"
-  maintainer: "luis.araujo@collabora.co.uk"
-  scope:
-  - functional
-  devices:
-  - i386
-  environment:
-  - lava-test-shell
-
-install:
-  deps:
-  - busybox
-  - apertis-tests-apparmor-gstreamer1.0
-  - apertis-tests-apparmor-report
-
-run:
-  steps:
-  - echo -n | sudo tee /var/log/audit/audit.log
-  # Use a timeout of 300s because some of the media file handling on embedded
-  # devices can be slow. This test is purely for AppArmor, not multimedia
-  # performance.
-  - common/run-test-in-systemd --name run-test-sh --timeout 300 --user=user /usr/lib/apertis-tests/apparmor/gstreamer1.0/lava_run_tests
-  - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
-
-parse:
-  fixupdict:
-    FAILED: fail
-    PASSED: pass
-    SKIP: skip
-  pattern: ^(?P<test_case_id>.+)\.sh:\s*(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)\s*-
-

apparmor/apparmor-libreoffice.yaml

-metadata:
-  name: apparmor-libreoffice
-  format: "Lava-Test-Shell Test Definition 1.0"
-  description: "Tests that the libreoffice profile works properly does not allow arbitrary 
-                reading of files in the home directory."
-  maintainer: "luis.araujo@collabora.co.uk"
-  scope:
-  - functional
-  devices:
-  - i386
-  environment:
-  - lava-test-shell
-
-install:
-  deps:
-  - busybox
-  - apertis-tests-apparmor-libreoffice
-  - apertis-tests-apparmor-report
-
-run:
-  steps:
-  - echo -n | sudo tee /var/log/audit/audit.log
-  - common/run-test-in-systemd --name run-test-libreoffice --timeout 300 /usr/lib/apertis-tests/apparmor/libreoffice/test-libreoffice
-  - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
-
-parse:
-  pattern: ^(?P<test_case_id>[\w\-\.]+):\s(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)$
-

apparmor/apparmor-profile-sanity-check.sh

@@ -34,9 +34,9 @@ find "/etc/apparmor.d" "/var/lib/apparmor/profiles" -type f | {
   exit $e
 } || e=$?
 
-if [ "x$e" = x0 ]; then
+if [ "$e" = 0 ]; then
   echo "pass:found_at_least_one"
-elif [ "x$e" = x101 ]; then
+elif [ "$e" = 101 ]; then
   echo "fail:found_at_least_one"
 else
   echo "unknown:found_at_least_one"

apparmor/apparmor-pulseaudio.yaml

-metadata:
-  name: apparmor-pulseaudio
-  format: "Lava-Test-Shell Test Definition 1.0"
-  description: "Test that AppArmor profiles provided in PulseAudio work without Auditd
-                complaining"
-  maintainer: "luis.araujo@collabora.co.uk"
-  scope:
-  - functional
-  devices:
-  - i386
-  environment:
-  - lava-test-shell
-
-install:
-  deps:
-  - busybox
-  - apertis-tests-apparmor-pulseaudio
-  - apertis-tests-apparmor-report
-
-run:
-  steps:
-  - echo -n | sudo tee /var/log/audit/audit.log
-  - common/run-test-in-systemd --name run-test-pulseaudio --timeout 90 /usr/lib/apertis-tests/apparmor/pulseaudio/test-pulseaudio
-  - sudo cat /var/log/audit/audit.log | aa_log_extract_tokens.pl PERMITTING REJECTING
-
-parse:
-  pattern: ^(?P<test_case_id>[\w\-\.]+):\s(?P<result>PASS|pass|FAIL|fail|SKIP|skip|UNKNOWN|unknown)$
-

apparmor/apparmor-session-lockdown-no-deny-before-reboot.yaml

-metadata:
-  name: apparmor-session-lockdown-no-deny-before-reboot
-  format: "Lava-Test-Shell Test Definition 1.0"
-  description: "Prepare for apparmor-session-lockdown-no-deny test"
-  maintainer: "apertis-tests maintainers <apertis-tests@apertis.org>"
-  scope:
-  - functional
-  devices:
-  - i386
-  environment:
-  - lava-test-shell
-
-install:
-  deps:
-  - busybox
-  - apertis-tests-apparmor-report
-  - apparmor-utils
-
-run:
-  steps:
-  - common/run-test-in-systemd --name apparmor-session-lockdown-no-deny-before-reboot --timeout 90 apparmor/session-lockdown/no-deny before-reboot
-
-parse:
-  pattern: ^(?P<result>ok|not ok)\s+\d+\s+(-\s*)?(?P<test_case_id>[\w\-\.]+).*$
-  fixupdict:
-    ok: pass
-    not ok: fail