diff --git a/renderer/images/3d-rendering-reference.png b/renderer/images/3d-rendering-reference.png new file mode 100644 index 0000000000000000000000000000000000000000..2de32899e3924bc72f6b8443f064ea52dbe424a1 Binary files /dev/null and b/renderer/images/3d-rendering-reference.png differ diff --git a/renderer/images/Actor-tiles-01.png b/renderer/images/Actor-tiles-01.png new file mode 100644 index 0000000000000000000000000000000000000000..e4b610ea520d5a19a660ff119f7f5ed5ab5c244f Binary files /dev/null and b/renderer/images/Actor-tiles-01.png differ diff --git a/renderer/images/Actor-tiles-02.png b/renderer/images/Actor-tiles-02.png new file mode 100644 index 0000000000000000000000000000000000000000..5f23a9da23a3022f6a042b7a55dd22122ef92ad8 Binary files /dev/null and b/renderer/images/Actor-tiles-02.png differ diff --git a/renderer/images/Actor-tiles-03.png b/renderer/images/Actor-tiles-03.png new file mode 100644 index 0000000000000000000000000000000000000000..007eda55bd9101bc88e9e7f63bc13ae6ac635a16 Binary files /dev/null and b/renderer/images/Actor-tiles-03.png differ diff --git a/renderer/images/Actor-tiles-04.png b/renderer/images/Actor-tiles-04.png new file mode 100644 index 0000000000000000000000000000000000000000..fbf08e5d08540ab569ca62f3bc775e960b5e5975 Binary files /dev/null and b/renderer/images/Actor-tiles-04.png differ diff --git a/renderer/images/Actor-tiles-05.png b/renderer/images/Actor-tiles-05.png new file mode 100644 index 0000000000000000000000000000000000000000..2ecb2cb9e9ec87b9e00010ca02176826f13091a6 Binary files /dev/null and b/renderer/images/Actor-tiles-05.png differ diff --git a/renderer/render-test-case b/renderer/render-test-case index 5f66570d418773f99b54f401e38a576513add897..8713789c33a9f7bec8ddcec408a5837c31a10150 100755 --- a/renderer/render-test-case +++ b/renderer/render-test-case @@ -23,6 +23,8 @@ TEMPLATE_DIR="." # expected: # - Start line with '$' for commands. # - Start line with '>' for command output. +# - Start line with '@' to attach image. +# - Start line with '~' to add web link. # - Everything else is a comment. # # run: steps: (only for automated tests) @@ -39,17 +41,21 @@ def parse_list(lines, automated_run=False): if automated_run: if sline[0] == '#': # Remove the '#' with the slice [1:] - processed_lines.append((sline[1:], '', '')) + processed_lines.append((sline[1:], '', '', '', '')) else: # Add the '$ ' in the line for the command - processed_lines.append(('', '$ '+sline, '')) + processed_lines.append(('', '$ '+sline, '', '', '')) else: if sline[0] == '$': - processed_lines.append(('', sline, '')) + processed_lines.append(('', sline, '', '', '')) elif sline[0] == '>': - processed_lines.append(('', '', sline[1:].split('\n'))) + processed_lines.append(('', '', sline[1:].split('\n'), '', '')) + elif sline[0] == '@': + processed_lines.append(('', '', '', sline[1:], '')) + elif sline[0] == '~': + processed_lines.append(('', '', '', '', sline[1:])) else: - processed_lines.append((sline, '', '')) + processed_lines.append((sline, '', '', '', '')) return processed_lines @@ -190,7 +196,13 @@ if '__main__' == __name__: c += 1 print("Total of test cases processed:", c) + # Copy CSS and images directory cssdir = os.path.join(directory, 'css') if not (os.path.exists(cssdir) and os.path.isdir(cssdir)): print("Copying css style to", cssdir) shutil.copytree('css/', cssdir) + + imagesdir = os.path.join(directory, 'images') + if not (os.path.exists(imagesdir) and os.path.isdir(imagesdir)): + print("Copying images to", imagesdir) + shutil.copytree('images/', imagesdir) diff --git a/renderer/templates/index.html b/renderer/templates/index.html index 6c15088a4b725b774474947c0f0bb73cc0e1ee5e..6cf942f1025e8eda565f776ddd79496db54cc3fb 100644 --- a/renderer/templates/index.html +++ b/renderer/templates/index.html @@ -43,9 +43,10 @@ {% if pkgname %}{{ macros.ostree_preconditions(pkgname) }}{% endif %} {% if packages_list %}{{ macros.install_packages(packages_list) }}{% endif %} {% if libname %}{{ macros.modules_preconditions(libname) }}{% endif %} - {% for comment, command, output in pre_conditions %} + {% for comment, command, output, _, link in pre_conditions %} {% if comment %}<li class="mb-sm-2">{{ comment|e }}</li>{% endif %} {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %} + {% if link %}<p><a href="{{ link }}">{{ link }}</a></p>{% endif %} {% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %} {% endfor %} </ol> @@ -54,7 +55,7 @@ <hr /> <h4>Execution Steps</h4> <ol> - {% for comment, command, output in run_steps %} + {% for comment, command, output, _, link in run_steps %} {% if comment %}<li class="mb-sm-2">{{ comment|e }}</li>{% endif %} {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %} {% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %} @@ -63,17 +64,19 @@ <hr /> <h4>Expected</h4> - {% for comment, command, output in expected %} + {% for comment, command, output, image, link in expected %} {% if comment %}<p class="mt-sm-3">{{ comment|e }}</p>{% endif %} {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %} {% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %} + {% if image %}<img src="images/{{ image }}" class="img-fluid">{% endif %} + {% if link %}<p><a href="{{ link }}">{{ link }}</a></p>{% endif %} {% endfor %} {% if post_conditions %} <hr /> <h4>Post Conditions</h4> <ol> - {% for comment, command, output in post_conditions %} + {% for comment, command, output, _, link in post_conditions %} {% if comment %}<li class="mb-sm-2">{{ comment|e }}</li>{% endif %} {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %} {% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %} @@ -89,10 +92,11 @@ <div class="card-body"> <h4>Notes</h4> <ul> - {% for comment, command, output in notes %} + {% for comment, command, output, _, link in notes %} {% if comment %}<li class="mb-sm-2">{{ comment|e }}</li>{% endif %} {% if command %}<p><kbd>{{ command|e }}</kbd></p>{% endif %} {% if output %}{% for l in output %}<p class="mb-sm-0 pl-sm-3"><samp>{{ l|e }}</samp></p>{% endfor %}{% endif %} + {% if link %}<p><a href="{{ link }}">{{ link }}</a></p>{% endif %} {% endfor %} </ul> </div> diff --git a/tests-cases/canterbury.yaml b/tests-cases/canterbury.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c581cc3a5189507401d2625124c494bd5155946 --- /dev/null +++ b/tests-cases/canterbury.yaml @@ -0,0 +1,30 @@ +metadata: + name: canterbury + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: unit + exec-type: automated + priority: critical + maintainer: "Apertis Project" + description: "Test for Canterbury library." + + macro_ostree_preconditions: canterbury + + expected: + - "The output is extremely verbose. However, the relevant lines have the + following structure:" + - | + >==== TEST <testname> passed as <username> ==== + ==== TEST <testname> failed as <username> ==== + +run: + steps: + - common/run-test-in-systemd --timeout=900 --user=user --name=canterbury -- ./sadt.sh --verbose + +parse: + pattern: '^(?P<test_case_id>[a-zA-Z0-9_\-\./]+)\s...\s(?P<result>ok|FAIL|SKIP)' + fixupdict: + ok: pass + FAIL: fail + SKIP: skip diff --git a/tests-cases/image-bootable.yaml b/tests-cases/image-bootable.yaml index d6db5edb6eff99a5b5587315ebea5f3c111724b8..c53f3265c175d3edc566b9bba0740f087783545f 100644 --- a/tests-cases/image-bootable.yaml +++ b/tests-cases/image-bootable.yaml @@ -12,13 +12,15 @@ metadata: pre-conditions: - "For SDK image: Remember, we're now requiring VirtualBox 4.2.2 for tests - using Guest Additions. Please, read https://wiki.apertis.org/Docs/VirtualBox_Guest_AdditionsDocs/VirtualBox_Guest_Additions to update your setup." - - "Download the virtual machine image for the latest SDK release from - https://images.apertis.org/" + using Guest Additions. To update setup, please read:" + - "~https://wiki.apertis.org/Docs/VirtualBox_Guest_Additions" + - "Download the virtual machine image for the latest SDK release from:" + - "~https://images.apertis.org/" - "For Target/Development images:" - - "Download latest target/development image released from - https://images.apertis.org/" - - "Write image to a SD card as explained at: https://wiki.apertis.org/System_Image_Setup#Target_and_development_images" + - "Download latest target/development image released from:" + - "~https://images.apertis.org/" + - "Write image to a SD card as explained at:" + - "~https://wiki.apertis.org/System_Image_Setup#Target_and_development_images" expected: - "Image boots either to a terminal or graphical environment." diff --git a/tests-cases/image-gui-start.yaml b/tests-cases/image-gui-start.yaml index ef27310ba21e0dbe3bc61f013d1a88d4b1284a1e..2fd67815d789fcc2ee3fa7509fc56b3b70943113 100644 --- a/tests-cases/image-gui-start.yaml +++ b/tests-cases/image-gui-start.yaml @@ -12,13 +12,15 @@ metadata: pre-conditions: - "For SDK image: Remember, we're now requiring VirtualBox 4.2.2 for tests - using Guest Additions. Please, read https://wiki.apertis.org/Docs/VirtualBox_Guest_AdditionsDocs/VirtualBox_Guest_Additions to update your setup." - - "Download the virtual machine image for the latest SDK release from - https://images.apertis.org/" + using Guest Additions. To update setup, please read:" + - "~https://wiki.apertis.org/Docs/VirtualBox_Guest_Additions" + - "Download the virtual machine image for the latest SDK release from:" + - "~https://images.apertis.org/" - "For Target/Development images:" - - "Download latest target/development image released from - https://images.apertis.org/" - - "Write image to a SD card as explained at: https://wiki.apertis.org/System_Image_Setup#Target_and_development_images" + - "Download latest target/development image released from:" + - "~https://images.apertis.org/" + - "Write image to a SD card as explained at:" + - "~https://wiki.apertis.org/System_Image_Setup#Target_and_development_images" expected: - "The image correctly starts the respective graphical interface." diff --git a/tests-cases/sdk-ide-build-run-program.yaml b/tests-cases/sdk-ide-build-run-program.yaml index 8df3ab7ad12f9e3cee445c78a097fb708a6aa087..42c73027252dae16405d365e91d2bf22a85b1553 100644 --- a/tests-cases/sdk-ide-build-run-program.yaml +++ b/tests-cases/sdk-ide-build-run-program.yaml @@ -12,9 +12,10 @@ metadata: pre-conditions: - "For SDK image: Remember, we're now requiring VirtualBox 4.2.2 for tests - using Guest Additions. Please, read https://wiki.apertis.org/Docs/VirtualBox_Guest_AdditionsDocs/VirtualBox_Guest_Additions to update your setup." - - "Download the virtual machine image for the latest SDK release from - https://images.apertis.org/" + using Guest Additions. To update setup, please read:" + - "~https://wiki.apertis.org/Docs/VirtualBox_Guest_Additions" + - "Download the virtual machine image for the latest SDK release from:" + - "~https://images.apertis.org/" expected: - "The Console tab at the bottom should show the output of the program diff --git a/tests-cases/sdk-vb-fullscreen.yaml b/tests-cases/sdk-vb-fullscreen.yaml index 2a3c9e6d7ec57a4b305dbdfc8fe26149b8a1084e..70cf17fcb6158b1fdca6504d9d3cab666b59710a 100644 --- a/tests-cases/sdk-vb-fullscreen.yaml +++ b/tests-cases/sdk-vb-fullscreen.yaml @@ -13,9 +13,10 @@ metadata: pre-conditions: - "For SDK image: Remember, we're now requiring VirtualBox 4.2.2 for tests - using Guest Additions. Please, read https://wiki.apertis.org/Docs/VirtualBox_Guest_AdditionsDocs/VirtualBox_Guest_Additions to update your setup." - - "Download the virtual machine image for the latest SDK release from - https://images.apertis.org/" + using Guest Additions. To update setup, please read:" + - "~https://wiki.apertis.org/Docs/VirtualBox_Guest_Additions" + - "Download the virtual machine image for the latest SDK release from:" + - "~https://images.apertis.org/" expected: - "The screen is correctly resized when switching to fullscreen." diff --git a/tests-cases/telepathy-ring.yaml b/tests-cases/telepathy-ring.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e076d36b7794cc5467c7bdbe1a835aaf38210a69 --- /dev/null +++ b/tests-cases/telepathy-ring.yaml @@ -0,0 +1,50 @@ +metadata: + name: telepathy-ring + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: manual + priority: medium + maintainer: "Apertis Project" + description: "Support making and receiving phone calls in Telepathy manually, + with a bluetooth-paired phone." + + resources: + - "A Bluetooth adapter" + - "A Bluetooth-enabled phone" + + macro_install_packages_preconditions: ofono-tests telepathy-ring apertis-tests + gir1.2-telepathyglib-0.12 + telepathy-mission-control-5 + pre-conditions: + - "Start simple agent:" + - $ /usr/lib/chaiwala-tests/bluez/simple-agent -y + - "Pair both devices initiating from the phone side" + - "Pay attention to the simple-agent window as it requires input during + the pairing process." + + expected: + - "It should show PASSED:" + - | + >telepathy-ring: PASSED + +run: + steps: + - "Pair phone. First scan for devices:" + - $ hcitool scan + - "Determine what the ID for your phone is. This will show a list of modems, + from which you can get the ID of your phone (which starts with a '/'), to + be used in the next 2 commands:" + - $ /usr/share/ofono/tests/list-modems + - "Power up modem:" + - $ /usr/share/ofono/tests/enable-modem $your_phone_id + - $ /usr/share/ofono/tests/online-modem $your_phone_id + - "Add a telepathy-ring account:" + - $ mc-tool add ring/tel tp-ring-test + - "Enable account:" + - $ mc-tool enable ring/tel/account0 + - "Test making a call:" + - $ /usr/lib/chaiwala-tests/telepathy/telepathy-ring call $number_to_call + - "Remove temporary account:" + - $ mc-tool remove ring/tel/account0 diff --git a/tests-cases/video-animation-on-boot.yaml b/tests-cases/video-animation-on-boot.yaml index f8e2446949ec30497cee38dc29a692b7974c103b..6e2e47fbb21865340db7d2b259fceb5e4bb76fa6 100644 --- a/tests-cases/video-animation-on-boot.yaml +++ b/tests-cases/video-animation-on-boot.yaml @@ -14,8 +14,8 @@ metadata: expected: - "Plymouth shows the fade-in theme with a chaiwala logo and a set of leaves - should fade in. Check example at: - https://wiki.apertis.org/mediawiki/images/7/7d/Boot.gif" + should fade in. Check example at:" + - "~https://wiki.apertis.org/mediawiki/images/7/7d/Boot.gif" run: steps: diff --git a/tests-cases/webkit2gtk-ac-3d-rendering.yaml b/tests-cases/webkit2gtk-ac-3d-rendering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..017d209cead7f2c2cc1b5aea5c86899398e0e590 --- /dev/null +++ b/tests-cases/webkit2gtk-ac-3d-rendering.yaml @@ -0,0 +1,32 @@ +metadata: + name: webkit2gtk-ac-3d-rendering + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: manual + priority: low + maintainer: "Apertis Project" + description: "Test that 3D transforms are applied correctly." + + resources: + - "A monitor connected to the testing target." + + macro_install_packages_preconditions: webkit2gtk-testing + + expected: + - "This test shows 4 squares with 3D rotation, they should look like they have + depth; see the correct image and what it would look like if 3D rendering + was not working below:" + - "@3d-rendering-reference.png" + + notes: + - "Not working under X11 or VirtualBox, see: + https://phabricator.apertis.org/T2625" + +run: + steps: + - "Run GtkClutterLauncher with the URL listed in the resources section as + argument:" + - $ GtkClutterLauncher file:///usr/share/webkit2gtk/testing/LayoutTests/transforms/3d/hit-testing/rotated-hit-test2.html + - "Verify rendering matches one of the two references below." diff --git a/tests-cases/webkit2gtk-ac-animations.yaml b/tests-cases/webkit2gtk-ac-animations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75925a9124a9b42786edf2aff9c5615452149529 --- /dev/null +++ b/tests-cases/webkit2gtk-ac-animations.yaml @@ -0,0 +1,42 @@ +metadata: + name: webkit2gtk-ac-animations + format: "Apertis Test Definition 1.0" + image-type: any + image-arch: any + type: functional + exec-type: manual + priority: medium + maintainer: "Apertis Project" + description: "Test that Webkit2GTK actors created for representing layers + animate properly." + + resources: + - "A monitor connected to the testing target." + + macro_install_packages_preconditions: webkit2gtk-testing + + expected: + - "Note that the tests log the results to the HTML page as it runs. The + contents of the log are not to be taken into account for the purpose of + this test. A FAIL in the logs does not mean this test case failed." + - "In the first URI, the two squares will rotate for 10 seconds. While + rotating, they should have a blue border, which indicates they are AC + layers. The expected rendering can be seen in the following video:" + - "~https://www.apertis.org/static/AC-animation.ogv" + - "The second URI will again show two blue squares. Both will move to the + right and animate their opacity at the same time, so will go from blue to + white and back. They should have borders while animating. The expected + rendering can be seen in the following video:" + - "~https://www.apertis.org/static/AC-animation-keyframe-opacity.ogv" + - "If the result does not match the squares as seen in the videos above, the + test has failed." + +run: + steps: + - "Run the following command:" + - $ GtkClutterLauncher file:///usr/share/webkit2gtk/testing/LayoutTests/animations/simultaneous-start-transform.html + - "Verify animation match the reference rendering below." + - "Either close GtkClutterLauncher and run the following command, or just + copy/paste the URI into the existing window:" + - $ GtkClutterLauncher file:///usr/share/webkit2gtk/testing/LayoutTests/animations//missing-keyframe-properties.html + - "Verify animation match the reference rendering below." diff --git a/tests-cases/webkit2gtk-actor-tiles.yaml b/tests-cases/webkit2gtk-actor-tiles.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ef73d2055af966a2a32ee0eafc02613381d81523 --- /dev/null +++ b/tests-cases/webkit2gtk-actor-tiles.yaml @@ -0,0 +1,58 @@ +metadata: + name: webkit2gtk-actor-tiles + format: "Apertis Test Definition 1.0" + image-type: target + image-arch: any + type: functional + exec-type: manual + priority: medium + maintainer: "Apertis Project" + description: "Test implementation of actor-based tiled backing store in + webkit2gtk." + + resources: + - "Mouse or touchscreen." + + macro_install_packages_preconditions: webkit2gtk-testing + + expected: + - "This test will be considered a PASS if it works with at least one of mouse + or touchpad because this is not a test for the mouse or touchpad working." + - "The page will look like this after the first load:" + - "@Actor-tiles-01.png" + - "After clicking the first link the page will look like this:" + - "@Actor-tiles-02.png" + - "After clicking the second link the page will look like this:" + - "@Actor-tiles-03.png" + - "The new page that will be loaded looks like this:" + - "@Actor-tiles-04.png" + - "Going back should give you the same rendering you had before clicking the + link to the new page, except the link will be colored purple." + - "@Actor-tiles-05.png" + - "After clicking the link that will take you to the beginning of the page and + repeating the process, the renderings should be the same as the ones above." + - "Note that when a page starts loading some of the tiles may contain a + checker board; that is normal, but we plan to improve on it." + +run: + steps: + - "Launch the test application with each URL in resources with and without + CLUTTER_PAINT=paint-volumes:" + - $ env CLUTTER_PAINT=paint-volumes GtkClutterLauncher file:///usr/share/webkit2gtk/testing/actor-tiles-test.html + - "Check that the rendering of the page matches the reference in the results + section." + - "Click the link that will scroll the page a bit" + - "Check that the rendering of the page matches the reference in the results + section" + - "Click the link that will scroll to the very bottom" + - "Check that the rendering of the page matches the reference in the results + section" + - "Click the first link that will take you to a different page" + - "Check that the rendering of the page matches the reference in the results + section" + - "Click the back button of the browser" + - "Check that the rendering of the page matches the reference in the results + section" + - "Click the second link, that will send you to the bottom" + - "Follow the links again until you get to the end of the page again, always + making sure the rendering matches the reference rendering" diff --git a/tests-cases/webkit2gtk-aligned-scroll.yaml b/tests-cases/webkit2gtk-aligned-scroll.yaml index 382715fb5bf83827484f9cda48d3d510a676a31f..d31a8f2ea11eba4de5ffb4583243060e08b7f388 100644 --- a/tests-cases/webkit2gtk-aligned-scroll.yaml +++ b/tests-cases/webkit2gtk-aligned-scroll.yaml @@ -25,8 +25,8 @@ metadata: finger/mouse vertically." notes: - - "Check the following example video for the expected result: - https://wiki.apertis.org/static/Aligned-scroll.ogv" + - "Check the following example video for the expected result:" + - "~https://wiki.apertis.org/static/Aligned-scroll.ogv" - "Both mouse and touchscreen need to PASS for this test case to be considered a PASS. If either does not pass, then the test case has failed."