.gitlab-ci.yml

+stages:
+  - test
+  - generate
+
+variables:
+  osname: apertis
+  release: v2026dev2
+
+image: $CI_REGISTRY/infrastructure/${osname}-docker-images/${release}-testcases-builder
+
+include:
+  - /.gitlab-ci/templates-generate-tests.yml
+
+test-renderer:
+  stage: test
+  tags:
+    - lightweight
+  script:
+    - python3 -m unittest discover -v
+
+render-pages:
+  stage: test
+  tags:
+    - lightweight
+  script:
+    - ./atc -d ${release}/ --index-page test-cases/
+  artifacts:
+    paths:
+      - ${release}
+
+.fixedfunction:
+  variables:
+    type: fixedfunction
+
+.hmi:
+  variables:
+    type: hmi
+
+.basesdk:
+  variables:
+    type: basesdk
+
+.sdk:
+  variables:
+    type: sdk
+
+.amd64-uefi:
+  variables:
+    architecture: amd64
+    board: uefi
+
+.amd64-sdk:
+  variables:
+    architecture: amd64
+    board: sdk
+
+.armhf-uboot:
+  variables:
+    architecture: armhf
+    board: uboot
+
+.arm64-uboot:
+  variables:
+    architecture: arm64
+    board: uboot
+
+.arm64-rpi64:
+  variables:
+    architecture: arm64
+    board: rpi64
+
+.generate-tests-apt:
+  extends: .generate-tests
+  tags:
+    - lightweight
+  stage: generate
+  variables:
+    deployment: apt
+    image_name: ${osname}_${release}-${type}-${architecture}-${board}
+    osname: apertis
+    release: v2026dev2
+    base_url: https://images.apertis.org
+    image_path: weekly/v2026dev2
+    image_buildid: "fakebuildid"
+
+.generate-tests-ostree:
+  extends: .generate-tests
+  tags:
+    - lightweight
+  stage: generate
+  variables:
+    deployment: ostree
+    image_name: ${osname}_ostree_${release}-${type}-${architecture}-${board}
+    image_bundle: ${osname}_ostree_${release}-${type}-${architecture}-${board}
+    osname: apertis
+    release: v2026dev2
+    base_url: https://images.apertis.org
+    image_path: weekly/v2026dev2
+    image_buildid: "fakebuildid"
+
+# !!!
+# All generate-tests-* jobs defined below are not really used,
+# they are intended to test that the jobs can be generated.
+# !!!
+
+generate-tests-apt-amd64-fixedfunction-uefi:
+  extends:
+    - .generate-tests-apt
+    - .amd64-uefi
+    - .fixedfunction
+
+generate-tests-apt-armhf-fixedfunction-uboot:
+  extends:
+    - .generate-tests-apt
+    - .armhf-uboot
+    - .fixedfunction
+
+generate-tests-apt-arm64-fixedfunction-uboot:
+  extends:
+    - .generate-tests-apt
+    - .arm64-uboot
+    - .fixedfunction
+
+generate-tests-apt-arm64-fixedfunction-rpi64-rpi4:
+  extends:
+    - .generate-tests-apt
+    - .arm64-rpi64
+    - .fixedfunction
+
+generate-tests-apt-amd64-hmi-uefi:
+  extends:
+    - .generate-tests-apt
+    - .amd64-uefi
+    - .hmi
+
+generate-tests-apt-armhf-hmi-uboot:
+  extends:
+    - .generate-tests-apt
+    - .armhf-uboot
+    - .hmi
+
+generate-tests-apt-arm64-hmi-rpi64-rpi4:
+  extends:
+    - .generate-tests-apt
+    - .arm64-rpi64
+    - .hmi
+
+generate-tests-apt-amd64-basesdk-sdk:
+  extends:
+    - .generate-tests-apt
+    - .amd64-sdk
+    - .basesdk
+
+generate-tests-apt-amd64-sdk-sdk:
+  extends:
+    - .generate-tests-apt
+    - .amd64-sdk
+    - .sdk
+
+generate-tests-ostree-amd64-fixedfunction-uefi:
+  extends:
+    - .generate-tests-ostree
+    - .amd64-uefi
+    - .fixedfunction
+
+generate-tests-ostree-armhf-fixedfunction-uboot:
+  extends:
+    - .generate-tests-ostree
+    - .armhf-uboot
+    - .fixedfunction
+
+generate-tests-ostree-arm64-fixedfunction-uboot:
+  extends:
+    - .generate-tests-ostree
+    - .arm64-uboot
+    - .fixedfunction
+
+generate-tests-ostree-arm64-fixedfunction-rpi64-rpi4:
+  extends:
+    - .generate-tests-ostree
+    - .arm64-rpi64
+    - .fixedfunction
+  variables:
+    image_bundle: ${osname}_ostree_${release}-${type}-${architecture}-uboot
+
+generate-tests-ostree-amd64-hmi-uefi:
+  extends:
+    - .generate-tests-ostree
+    - .amd64-uefi
+    - .hmi
+
+generate-tests-ostree-arm64-hmi-rpi64-rpi4:
+  extends:
+    - .generate-tests-ostree
+    - .arm64-rpi64
+    - .hmi
+  variables:
+    image_bundle: ${osname}_ostree_${release}-${type}-${architecture}-uboot

.gitlab-ci/image-pipeline-base.yml

+.common-image-job-base:
+  image: $CI_REGISTRY/infrastructure/${OSNAME}-docker-images/${APERTIS_RELEASE}-image-builder
+  variables:
+    TEST_PATH: /tmp/tests
+    TEST_REPO_URL: https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/tests/${OSNAME}-test-cases.git
+    RECIPES_PATH: /tmp/recipes
+    RECIPES_REPO_URL: https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/infrastructure/${OSNAME}-image-recipes.git
+  stage: build
+  timeout: 2h
+  rules:
+  - when: on_success
+  before_script:
+    - JOB_GENERATOR_BRANCH=${JOB_GENERATOR_BRANCH:-${OSNAME}/${APERTIS_RELEASE}}
+    - git clone --depth "${GIT_DEPTH:-1}" -b "$JOB_GENERATOR_BRANCH" "${TEST_REPO_URL}" "$TEST_PATH"
+    - git clone --depth "${GIT_DEPTH:-1}" -b "${OSNAME}/${APERTIS_RELEASE}" "${RECIPES_REPO_URL}" "$RECIPES_PATH"
+    - >
+      "$TEST_PATH/extract-variables.py" < "$RECIPES_PATH/.gitlab-ci.yml" > image.env
+    - . image.env
+    - >
+      "$TEST_PATH/extract-variables.py" ".$TYPE" < "$RECIPES_PATH/.gitlab-ci.yml" > pipeline.env
+    - . pipeline.env
+    - |
+      if [[ ${debosarguments} =~ "-t devrootpack:" ]]
+      then
+        DEVROOT_ARCH=armhf
+        devroot_ospack_name="ospack_${RELEASE}-${DEVROOT_ARCH}-devroot_${BUILDID}"
+        wget --progress=dot:mega ${image_url_prefix_main}/daily/${RELEASE}/${BUILDID}/${DEVROOT_ARCH}/devroot/${devroot_ospack_name}.tar.gz
+        debosarguments="$debosarguments -t devrootpack:${devroot_ospack_name}.tar.gz"
+      fi
+    - ospack_name="ospack_${RELEASE}-${ARCH}-${TYPE}_${BUILDID}"
+    - wget --progress=dot:mega ${image_url_prefix_main}/daily/${RELEASE}/${BUILDID}/${ARCH}/${TYPE}/${ospack_name}.tar.gz
+    - local_repo_path=$PWD/package-repo/$ARCH
+
+
+.image-job-base:
+  extends: .common-image-job-base
+  script:
+    - echo ${debosarguments}
+    - debos ${debosarguments}
+        --show-boot
+        -t architecture:${ARCH}
+        -t osname:${OSNAME}
+        -t mirror:${mirror}
+        -t type:${TYPE}
+        -t sbc:${SBC}
+        -t ospack:${ospack_name}
+        -t suite:$RELEASE
+        -t timestamp:${PIPELINE_VERSION}
+        -t image:${IMAGE_NAME}
+        -t local_repo_path:${local_repo_path}
+        -t extra_packages:"${BIN_PKG_LIST}"
+        "$RECIPES_PATH/image-${BOARD}.yaml"
+    - echo "BUILT_IMAGE_URL=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/artifacts/${IMAGE_NAME}.img" | tee -a build.env
+  artifacts:
+    reports:
+      dotenv: build.env
+    paths:
+      - '*.img.*'
+    expire_in: 1 day
+
+.ostree-image-job-base:
+  extends: .common-image-job-base
+  variables:
+    branch: ${OSNAME}/${RELEASE}/${ARCH}-${BOARD}/${TYPE}
+    repo: repo-${ARCH}-${BOARD}-${TYPE}/
+    IMAGE_NAME: ${OSNAME}_ostree_${RELEASE}-${TYPE}-${ARCH}-${BOARD}_${BUILDID}
+  script:
+    - ostree_pull_url=${IMAGE_URL_PREFIX}/${ostree_path}
+    - ostree init --repo=${repo} --mode archive-z2
+    - echo ${debosarguments}
+    - debos ${debosarguments}
+        --show-boot
+        -t architecture:${ARCH}
+        -t osname:${OSNAME}
+        -t mirror:${mirror}
+        -t type:${TYPE}
+        -t sbc:${SBC}
+        -t board:${BOARD}
+        -t suite:$RELEASE
+        -t ospack:${ospack_name}
+        -t image:${IMAGE_NAME}
+        -t message:${RELEASE}-${TYPE}-${ARCH}-${BOARD}_${BUILDID}
+        -t ostree:${repo}
+        -t branch:${branch}
+        -t collection_id:${collection_id}
+        -t local_repo_path:${local_repo_path}
+        -t extra_packages:"${BIN_PKG_LIST}"
+        "$RECIPES_PATH/ostree-commit.yaml"
+    - debos ${debosarguments}
+        --show-boot
+        -t architecture:${ARCH}
+        -t osname:${OSNAME}
+        -t mirror:${mirror}
+        -t type:${TYPE}
+        -t sbc:${SBC}
+        -t board:$BOARD
+        -t ospack:${ospack_name}
+        -t suite:$RELEASE
+        -t timestamp:${PIPELINE_VERSION}
+        -t image:${IMAGE_NAME}
+        -t ostree:${repo}
+        -t branch:${branch}
+        -t local_repo_path:${local_repo_path}
+        -t collection_id:${collection_id}
+        "$RECIPES_PATH/ostree-image-${BOARD}.yaml"
+    - echo "BUILT_IMAGE_URL=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/jobs/${CI_JOB_ID}/artifacts/${IMAGE_NAME}.img" | tee -a build.env
+  artifacts:
+    reports:
+      dotenv: build.env
+    paths:
+      - '*.img.*'
+      - '*.commit.*'
+    expire_in: 1 day

.gitlab-ci/templates-generate-tests.yml

+######
+# stage: generate tests
+
+.generate-tests:
+  image: $CI_REGISTRY/infrastructure/${osname}-docker-images/${release}-image-builder
+  variables:
+    release: v2021
+    # images are fetched from
+    # {{baseurl}}/{{image_path}}/{{image_buildid}}/{{arch}}/{{type}}/{{image_name}}.img.gz
+    # where `arch` and `type` are derived from `profile_name`
+    baseurl: https://images.apertis.org
+    image_name: apertis_v2023dev1-minimal-amd64-uefi_v2023dev1.1
+    image_buildid: "v2023dev1.1"
+    image_path: release/v2023dev1
+    source_commit: $CI_COMMIT_SHA
+    source_job: $CI_JOB_URL
+    source_pipeline: $CI_PIPELINE_URL
+    source_project: $CI_PROJECT_URL
+    source_ref: $CI_COMMIT_REF_NAME
+    # source_wip:true means that we skip creating tasks on Phab on test
+    # failures, and that in general the results should not impact the overall
+    # health of the project
+    source_wip: "true"
+    visibility: public
+  script:
+    - env --null | sort -z | tr '\0' '\n'
+    - ./generate-jobs.py
+        -d lava/devices.yaml
+        --config lava/config.yaml
+        --release ${release}
+        --arch ${architecture}
+        --board ${board}
+        --osname ${osname}
+        --type ${type}
+        --date ${image_buildid}
+        --deployment ${deployment}
+        --name ${image_name}
+        -t "baseurl:${baseurl}"
+        -t "image_bundle:${image_bundle}"
+        -t "imgpath:${image_path}"
+        -t "source_commit:${source_commit}"
+        -t "source_job:${source_job}"
+        -t "source_pipeline:${source_pipeline}"
+        -t "source_project:${source_project}"
+        -t "source_ref:${source_ref}"
+        -t "source_wip:${source_wip}"
+        -t "visibility:${visibility}"
+        --verbose
+        --metadata-file metadata.json
+    - ./generate-test-pipeline.py
+        --include .gitlab-ci/test-pipeline-base.yml
+        --output child-pipeline.yaml
+        --parent-pipeline $CI_PIPELINE_ID
+        --current-job-name $CI_JOB_NAME
+        --test-job-base '.test-job-base'
+        job-${image_name}-*.yaml
+
+  artifacts:
+    paths:
+      - ./job-${osname}*.yaml
+      - ./metadata.json
+      - ./child-pipeline.yaml
+    expire_in: 1 day
+  rules:
+    - if: '$CI_MERGE_REQUEST_ID'
+      when: never
+    - when: on_success

.gitlab-ci/test-pipeline-base.yml

+.test-job-base:
+  stage: deploy
+  tags:
+    - lava-runner
+  artifacts:
+    when: always
+    paths:
+      - log.yaml
+    reports:
+      junit: "*.xml"

MANIFEST.in

+include atc_renderer/css/*
+include atc_renderer/images/*
+include atc_renderer/templates/*
+include atc_renderer/tests_files/*

README.md

-Repository for Apertis tests cases.
+# Apertis Tests Cases and Renderer
 
-# Format Rules/Notes
+This repository contains all the Apertis test cases written using the YAML format
+defined at the `format/` directory.
+
+The repository also contains the `renderer-test-case` program, which is used to
+generate the test case pages in HTML format from the YAML files.
+
+## Format Rules/Notes
+
+- The renderer heavily uses `lists` of strings to parse those into the different
+  HTML components. Each string in a list can be started with a special character
+  to get a specific format if it is supported by the YAML directive containing
+  such a string.
+
+  For the `pre-conditions`, `notes`, `expected`, `run.steps` (only manual
+  tests) and `post-conditions` directives, start a line with the following
+  characters to give the described format:
+
+    - `$`: Command line.
+    - `>`: Command output.
+    - `@`: Attach image.
+    - `~`: Add a web link.
+
+  For the `run.steps` field in the automated tests, only the following formatting
+  rules apply:
+
+    - `#`: Add a comment.
+    - Everything else is a command line.
 
 - Test cases with an `install.deps` directive will automatically add a
-  `pre conditions` section to the test case page with the list of packages to
+  `pre-conditions` section to the test case page with the list of packages to
   install specified in such a directive.
+
+## Macro helpers
+
+Some test cases share the same set of instructions, for those, some `macros` are
+available: `macro_ostree_preconditions` , `macro_install_packages_preconditions`,
+and `macro_modules_preconditions`. Please check the tests cases for example about
+how to use them.
+
+## Images
+
+Test cases can contain images. These are attached to a test case page starting a
+line in one of the special directives with the character `@` followed by the image
+name, then this image should be placed under the `renderer/images` directory.
+
+## Renderer
+
+The main program is named `atc` in the parent directory and can be directly invoked
+like:
+
+```
+$ ./atc --help
+```
+
+The above command will show the help.
+
+To render a single test case, use something like:
+
+```
+$ ./atc test-cases/<test-case-name>.yaml
+```
+
+That will create a `<test-case-name>.html` page.
+
+To render a complete set of test cases inside a directory, just pass a directory
+instead of a file to the command, and optionally pass a new directory path with
+the `-d` flag so all the newly generated test cases are saved there instead of the
+current directory:
+
+```
+$ ./atc test-cases/ -d apertis-test-cases-v0.1/
+```
+
+Now all the html tests cases should be available inside the
+`apertis-test-cases-v0.1/` directory.
+
+### Renderer (parser) behaviour
+
+- The renderer parser will always abort with a failure if a mandatory field is
+  missing.
+
+- The renderer parser will abort with a failure for those fields set to values
+  with incorrect types.
+
+- The renderer parser will show warning messages for unrecognized values of
+  fields with multiple choices (for example, image-type, execution-type).
+
+## Tests
+
+The module `renderer/tests.py` contains tests for the parser, renderer and
+different formatting methods. There are also a set of tests files located inside
+the `renderer/tests_files/` directory that are used by some of these tests.
+
+The unittest can be executed from the parent directory with:
+
+```
+$ python3 -m unittest discover -v
+```
+
+It is highly recommended to execute these tests if changes are applied to any
+of the `renderer` components since merge requests will only be accepted if all the
+tests pass.

atc

+#!/usr/bin/env python3
+###################################################################################
+# Main program to render test case files.
+# Create a HTML page from a YAML test case file.
+#
+# Copyright (C) 2018
+# Luis Araujo <luis.araujo@collabora.co.uk>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  US
+###################################################################################
+
+from atc_renderer import main
+from argparse import ArgumentParser
+
+if '__main__' == __name__:
+    cli_parser = ArgumentParser(description="atc (Test Cases Renderer)")
+    cli_parser.add_argument('-i', '--index-page', action='store_true',
+                            help="Create index page")
+    cli_parser.add_argument('-d', '--test-case-dir',
+                            help="Directory path for generated test cases")
+    cli_parser.add_argument('yaml_files',
+                            help="YAML file or files directory")
+    args = cli_parser.parse_args()
+
+    main(args.yaml_files, args.test_case_dir, args.index_page)
+
+    exit(0)

atc_renderer/__init__.py

+###################################################################################
+# Main program to render test case files.
+# Create a HTML page from a YAML test case file.
+#
+# Copyright (C) 2018
+# Luis Araujo <luis.araujo@collabora.co.uk>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  US
+###################################################################################
+
+import os
+import shutil
+import pkg_resources
+
+from atc_renderer.renderer import generate_test_case_page, \
+    generate_index_page, \
+    PYTHON_PKGNAME
+
+def copy_files(directory, dirname, dirpath, msg):
+    """
+    Only copy files if destination directory (dst_dir) is not in the cwd.
+    """
+    dst_dir = os.path.join(os.path.realpath(directory), dirname)
+    if not os.path.isdir(dst_dir):
+        print("Creating directory", dst_dir)
+        os.mkdir(dst_dir)
+
+    print(msg, dst_dir)
+    files = os.listdir(dirpath)
+    for f in files:
+        shutil.copy2(os.path.join(dirpath, f), dst_dir)
+
+
+def main(tc_files, tc_dir = None, index_page = False):
+    index_files = []
+    directory = os.getcwd()
+    if tc_dir:
+        directory = tc_dir
+        try:
+            os.mkdir(directory)
+        except FileExistsError:
+            print("Directory '{}' already exists".format(directory))
+        except e:
+            print("Error:", e)
+            exit(1)
+
+    if os.path.isfile(tc_files):
+        generate_test_case_page(tc_files, directory)
+        if index_page:
+            index_files.append(tc_files)
+    else:
+        c = 0
+        for root, _, files in os.walk(tc_files):
+            for f in files:
+                tc_file = os.path.join(root, f)
+                if os.path.isfile(tc_file):
+                    generate_test_case_page(tc_file, directory)
+                    if index_page:
+                        index_files.append(tc_file)
+                    c += 1
+        print("Total of test cases processed:", c)
+
+    # Copy CSS and images directory
+    css_dir = pkg_resources.resource_filename(PYTHON_PKGNAME, 'css/')
+    images_dir = pkg_resources.resource_filename(PYTHON_PKGNAME, 'images/')
+    copy_files(directory, 'css/', css_dir, "Copying css style to")
+    copy_files(directory, 'images/', images_dir, "Copying images to")
+
+    if index_files:
+        generate_index_page(index_files, directory)

atc_renderer/exceptions.py

+###################################################################################
+# Test case parser exceptions.
+#
+# Copyright (C) 2018
+# Luis Araujo <luis.araujo@collabora.co.uk>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  US
+###################################################################################
+
+
+class ParserError(Exception):
+    pass
+
+class ParserTypeError(ParserError):
+
+    def __init__(self, msg):
+        self._msg = msg
+
+class ParserMissingFieldError(ParserError):
+
+    def __init__(self, msg):
+        self._msg = msg