tests: Test the ci-package-builder pipeline

Set up a test repository and run the ci-package-builder pipeline on it by committing new changes, creating merge requests and monitoring the resulting pipelines.

The test currently checks:

• submitting non-release changes and landing them
• submitting release commits and landing them
• blocking commits to frozen stable branches

This pipeline needs some extra setup:

• a tests/dash> repository, forked from pkg/target/dash>, where we force-push changes, create merge requests and monitor pipelines
• the OBS_PEFIX CI variable to be set on tests/dash> to point to throwaway OBS branches to test the upload and monitorig jobs, for instance home:apertis-gitlab:branches:test:
• the GITLAB_CI_USER, GITLAB_CI_PASSWORD, OSC_USERNAME and OSC_PASSWORD CI variables to be set on tests/dash>, matching what it is used on the pkg/ projects
• the GITLAB_AUTH_TOKEN CI variable to be set on this repository to access the GitLab APIs used to issue MRs and monitor pipelines, and to push changes via git to the tests/dash> repository

Another extra bit of configuration was needed to be able to run the pipeline from non-default branches:

1. open https://gitlab.apertis.org/admin/application_settings/network#js-outbound-settings
2. enable Allow requests to the local network from web hooks and services
3. add gitlab.apertis.org to the allowlist

tests/test_packaging_pipelines.py

+#!/usr/bin/env python3
+
+import argparse
+import logging
+import pathlib
+import subprocess
+import time
+import urllib.parse
+
+import git
+import gitlab
+import osc.conf
+import osc.core
+
+SCRATCH_REPO_DIR = "/tmp/scratchrepo"
+TEST_BRANCH = "wip/test/fake"
+WAIT_AFTER_PUSH = 5
+
+
+def _get_branch_prefix(project):
+    sep = "/"
+    prefix = project.default_branch.split(sep)[0]
+    return prefix + sep
+
+
+def _find_stable_release_branch(project, prefix):
+    branches = (b.name for b in project.branches.list(all=True))
+    releases = (b for b in branches if b.startswith(prefix))
+    stable = (b for b in releases if b[-4:-1] != "dev" and b[-3:] != "pre")
+    latest = sorted(stable, reverse=True)[0]
+    return latest
+
+
+def _monitor_pipeline_for_completion(pipeline):
+    logging.info(f"monitoring pipeline {pipeline.web_url}")
+    while pipeline.status in ("running", "pending", "created"):
+        logging.debug(f"pipeline {pipeline.web_url} status is {pipeline.status}")
+        pipeline.refresh()
+        time.sleep(5)
+
+
+def _commit_and_push(gl, repo, filename, msg):
+    repo.index.add(filename)
+    actor = git.Actor(gl.user.name, gl.user.email)
+    commit = repo.index.commit(msg, author=actor, committer=actor)
+    logging.info(f"committed {commit.hexsha} to {TEST_BRANCH}")
+    pushes = repo.remotes.origin.push(TEST_BRANCH, force=True, verbose=True)
+    assert not any(p.flags & git.remote.PushInfo.ERROR for p in pushes)
+    time.sleep(
+        WAIT_AFTER_PUSH
+    )  # give GitLab some time to realize we pushed some changes
+    return commit
+
+
+class GitLabToOBSTester:
+    def __init__(self, reference, scratch, testing_pipeline_url):
+        self.gl = None
+        self.reference = reference
+        self.scratch = scratch
+        self.scratch_git = None
+        self.scratch_gitlab = None
+        self.stable_branch = None
+        self.testing_pipeline_url = testing_pipeline_url
+
+    def connect(self, gitlab_instance, gitlab_server_url, gitlab_auth_token, oscrc):
+        if gitlab_server_url:
+            logging.info(f'Connecting to the "{gitlab_server_url}" instance')
+            self.gl = gitlab.Gitlab(gitlab_server_url, private_token=gitlab_auth_token)
+        else:
+            logging.info(f'Connecting to the "{gitlab_instance}" configured instance')
+            self.gl = gitlab.Gitlab.from_config(gitlab_instance)
+        self.gl.auth()
+        osc.conf.get_config(override_conffile=oscrc)
+
+    def _scratch_set_properties(self):
+        logging.info(f"setting up properties on {self.scratch}, temporarily disable CI")
+        s = self.gl.projects.get(self.scratch)
+        self.scratch_gitlab = s
+        try:
+            s.branches.delete(TEST_BRANCH)
+        except gitlab.GitlabDeleteError:
+            pass
+        s.merge_requests_access_level = "enabled"
+        s.merge_method = "ff"
+        s.builds_access_level = "disabled"
+        s.lfs_enabled = True
+        s.save()
+
+    def _scratch_clone(self):
+        url = urllib.parse.urlparse(self.scratch_gitlab.http_url_to_repo)
+        password = urllib.parse.quote(self.gl.private_token, safe="")
+        netloc = f"oauth2:{password}@{url.netloc}"
+        url = url._replace(netloc=netloc)
+        logging.info(f"clone {url.geturl()} to {SCRATCH_REPO_DIR}")
+        self.scratch_git = git.Repo.clone_from(url.geturl(), SCRATCH_REPO_DIR)
+
+    def prepare_scratch(self):
+        self._scratch_set_properties()
+        self._scratch_clone()
+
+    def _scratch_set_variables(self):
+        obs_project_prefix = self._get_obs_test_project_prefix()
+        logging.info(f"force OBS projects under {obs_project_prefix}")
+        varname = "OBS_PREFIX"
+        vardata = {"key": varname, "value": obs_project_prefix}
+        try:
+            self.scratch_gitlab.variables.update(varname, vardata)
+        except gitlab.GitlabUpdateError:
+            self.scratch_gitlab.variables.create(vardata)
+
+    def reset_scratch(self):
+        logging.info(
+            f"resetting {self.scratch_gitlab.path_with_namespace} to {self.reference}"
+        )
+        scratch = self.scratch_git
+        reference = self.gl.projects.get(self.reference)
+        url = reference.http_url_to_repo
+        scratch.create_remote("reference", url)
+        scratch.remotes.reference.fetch()
+        logging.debug(
+            f"switching to the {TEST_BRANCH} branch as {scratch.head.commit.hexsha}"
+        )
+        scratch.create_head("wip/test/fake").checkout()
+        logging.debug("synchronizing branches")
+        default = reference.default_branch
+        prefix = _get_branch_prefix(reference)
+        self.stable_branch = _find_stable_release_branch(reference, prefix)
+        branches = [default, self.stable_branch, "pristine-lfs", "pristine-lfs-source"]
+        logging.info(f"synchronizing the {branches} branches")
+        for branchname in branches:
+            scratch.create_head(
+                branchname,
+                commit=scratch.remotes.reference.refs[branchname],
+                force=True,
+            )
+        logging.debug(
+            f"pushing {branches} to {self.scratch_gitlab.path_with_namespace}"
+        )
+        pushes = scratch.remotes.origin.push(branches, force=True, verbose=True)
+        assert not any(p.flags & git.remote.PushInfo.ERROR for p in pushes)
+        logging.debug(
+            f"setting the default branch on {self.scratch_gitlab.path_with_namespace} to {default} and re-enable CI"
+        )
+        self.scratch_gitlab.default_branch = default
+        self.scratch_gitlab.builds_access_level = "enabled"
+        self.scratch_gitlab.save()
+        self._scratch_set_variables()
+
+    def _get_obs_reference_project_name(self):
+        root = self.scratch_gitlab.default_branch.replace("/", ":")
+        component = "development"
+        filepath = pathlib.Path(SCRATCH_REPO_DIR, "debian/apertis/component")
+        if filepath.exists():
+            component = open(filepath).read().strip()
+        project = f"{root}:{component}"
+        return project
+
+    def _get_obs_test_project_prefix(self):
+        apiurl = osc.conf.config["apiurl"]
+        username = osc.conf.config["api_host_options"][apiurl]["user"]
+        test_project = f"home:{username}:branches:test:"
+        return test_project
+
+    def _get_obs_test_project_name(self):
+        apiurl = osc.conf.config["apiurl"]
+        prefix = self._get_obs_test_project_prefix()
+        reference_project = self._get_obs_reference_project_name()
+        test_project = prefix + reference_project
+        return test_project
+
+    def obs_prepare_work_areas(self):
+        logging.info("preparing work areas on OBS")
+        apiurl = osc.conf.config["apiurl"]
+        package = self.reference.split("/")[-1]
+        src_project = self._get_obs_reference_project_name()
+        target_project = self._get_obs_test_project_name()
+        msg = "Branch for testing the GitLab-to-OBS pipeline"
+        for suffix in ("", ":snapshots"):
+            logging.info(
+                f"branching {package} from {src_project + suffix} to {target_project + suffix}"
+            )
+            osc.core.branch_pkg(
+                apiurl,
+                src_project + suffix,
+                package,
+                target_project=target_project + suffix,
+                msg=msg,
+                force=True,
+            )
+
+    def point_gitlab_ci_here(self, ci_config_path):
+        logging.info(f"pointing to the CI definition at {ci_config_path}")
+        self.scratch_gitlab.ci_config_path = ci_config_path
+        self.scratch_gitlab.save()
+
+    def test_nonrelease_mr(self):
+        logging.info(f"testing an unreleased version on {TEST_BRANCH}")
+        assert self.scratch_git.active_branch.name == TEST_BRANCH
+        subprocess.run(
+            ["dch", "-i", "Fake changes while testing the GitLab-to-OBS pipeline"],
+            check=True,
+            cwd=SCRATCH_REPO_DIR,
+        )
+        msg = f"Test unreleased changes\n\nPipeline: {self.testing_pipeline_url}"
+        commit = _commit_and_push(self.gl, self.scratch_git, "debian/changelog", msg)
+        pipeline = self.scratch_gitlab.pipelines.list(
+            ref=TEST_BRANCH, sha=commit.hexsha
+        )[0]
+        _monitor_pipeline_for_completion(pipeline)
+        assert (
+            pipeline.status == "success"
+        ), f"submitted non-release pipeline {pipeline.web_url} didn't succeed: {pipeline.status}"
+        job_names = [j.name for j in pipeline.jobs.list()]
+        assert job_names == [
+            "build-source"
+        ], f"submitted non-release pipeline on unmerged changes expected different jobs: {job_names}"
+        target = self.scratch_gitlab.default_branch
+        mr = self.scratch_gitlab.mergerequests.create(
+            dict(
+                source_branch=TEST_BRANCH,
+                target_branch=target,
+                title="Test non-release",
+            )
+        )
+        logging.info(f"created non-release MR {mr.web_url}")
+        pipelines = [p for p in mr.pipelines() if p["sha"] == commit.hexsha]
+        assert (
+            len(pipelines) == 1
+        ), f"non-release MR has unexpected pipelines: {' '.join(p['web_url'] for p in pipelines)}"
+        assert mr.pipelines()[0]["id"] == pipeline.id
+        logging.info(f"landing {mr.web_url}")
+        time.sleep(WAIT_AFTER_PUSH)
+        mr.merge()
+        time.sleep(WAIT_AFTER_PUSH)
+        pipeline = self.scratch_gitlab.pipelines.list(
+            ref=self.scratch_gitlab.default_branch, sha=commit.hexsha
+        )[0]
+        _monitor_pipeline_for_completion(pipeline)
+        assert (
+            pipeline.status == "success"
+        ), f"landed non-release pipeline {pipeline.web_url} didn't succeed: {pipeline.status}"
+        job_names = [j.name for j in pipeline.jobs.list()]
+        assert job_names == [
+            "build-source",
+            "tag-release",
+            "upload",
+        ], f"landed non-release pipeline expected different jobs: {job_names}"
+        logging.info("Non-release MR landed successfully ✅")
+
+    def test_release_commit(self):
+        logging.info("submitting a release commit")
+        subprocess.run(["dch", "-r", "apertis"], check=True, cwd=SCRATCH_REPO_DIR)
+        msg = f"Test release commit\n\nPipeline: {self.testing_pipeline_url}"
+        commit = _commit_and_push(self.gl, self.scratch_git, "debian/changelog", msg)
+        pipeline = self.scratch_gitlab.pipelines.list(
+            ref=TEST_BRANCH, sha=commit.hexsha
+        )[0]
+        _monitor_pipeline_for_completion(pipeline)
+        assert (
+            pipeline.status == "success"
+        ), f"submitted release pipeline {pipeline.web_url} didn't succeed: '{pipeline.status}'"
+        job_names = [j.name for j in pipeline.jobs.list()]
+        assert job_names == [
+            "build-source"
+        ], f"submitted release pipeline on unmerged changes expected different jobs: {job_names}"
+        return commit
+
+    def test_release_mr_to_frozen_branch(self, commit):
+        logging.info("testing MR to frozen branch")
+        target = self.stable_branch
+        mr = self.scratch_gitlab.mergerequests.create(
+            dict(
+                source_branch=TEST_BRANCH,
+                target_branch=target,
+                title="Test release on frozen branch",
+            )
+        )
+        logging.info(f"created release MR {mr.web_url} to frozen branch {target}")
+        time.sleep(WAIT_AFTER_PUSH)
+        pipelines = {
+            p["ref"]: self.scratch_gitlab.pipelines.get(p["id"])
+            for p in mr.pipelines()
+            if p["sha"] == commit.hexsha
+        }
+        assert (
+            len(pipelines) == 2
+        ), f"unexpected pipelines: {p['web_url'] for p in pipelines}"
+        branch_pipeline = pipelines.pop(TEST_BRANCH)
+        mr_pipeline = next(iter(pipelines.values()))
+        _monitor_pipeline_for_completion(branch_pipeline)
+        _monitor_pipeline_for_completion(mr_pipeline)
+        assert (
+            branch_pipeline.status == "success"
+        ), f"release branch pipeline {branch_pipeline.web_url} to frozen branch didn't succeed: '{branch_pipeline.status}'"
+        assert (
+            mr_pipeline.status == "failed"
+        ), f"release MR pipeline {mr_pipeline.web_url} to frozen branch should have failed: '{mr_pipeline.status}'"
+        job_names = [j.name for j in mr_pipeline.jobs.list()]
+        assert job_names == [
+            "freeze-stable-branches"
+        ], f"submitted non-release pipeline expected different jobs: {job_names}"
+        mr.state_event = "close"
+        mr.save()
+        logging.info("MR on frozen branch blocked successfully ✅")
+
+    def test_release_mr_to_default_branch(self, commit):
+        logging.info("testing MR to default branch")
+        target = self.scratch_gitlab.default_branch
+        mr = self.scratch_gitlab.mergerequests.create(
+            dict(source_branch=TEST_BRANCH, target_branch=target, title="Test release")
+        )
+        logging.info(f"created release MR {mr.web_url} to default branch {target}")
+        pipelines = [p for p in mr.pipelines() if p["sha"] == commit.hexsha]
+        assert len(pipelines) == 1
+        assert pipelines[0]["ref"] == TEST_BRANCH
+        logging.info(f"landing {mr.web_url}")
+        time.sleep(WAIT_AFTER_PUSH)
+        mr.merge()
+        time.sleep(WAIT_AFTER_PUSH)
+        pipeline = self.scratch_gitlab.pipelines.list(
+            ref=self.scratch_gitlab.default_branch, sha=commit.hexsha
+        )[0]
+        _monitor_pipeline_for_completion(pipeline)
+        assert (
+            pipeline.status == "success"
+        ), f"landed release pipeline {pipeline.web_url} didn't succeed: '{pipeline.status}'"
+        job_names = [j.name for j in pipeline.jobs.list()]
+        assert job_names == [
+            "build-source",
+            "tag-release",
+            "upload",
+        ], f"landed release pipeline on merged changes expected different jobs: {job_names}"
+        logging.info("Release MR landed successfully ✅")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Test the GitLab-to-OBS pipeline")
+    parser.add_argument(
+        "--debug",
+        action="store_const",
+        dest="loglevel",
+        const=logging.DEBUG,
+        help="print debug information",
+    )
+    parser.add_argument(
+        "--scratch-repository",
+        type=str,
+        required=True,
+        help="the repository on which the pipeline is tested",
+    )
+    parser.add_argument(
+        "--reference-repository",
+        type=str,
+        required=True,
+        help="the scratch repository will be reset to the reference-repository contents",
+    )
+    parser.add_argument(
+        "--gitlab-instance",
+        type=str,
+        default="apertis",
+        help="get connection parameters from this configured instance",
+    )
+    parser.add_argument(
+        "--gitlab-auth-token", type=str, help="the GitLab authentication token"
+    )
+    parser.add_argument("--gitlab-server-url", type=str, help="the GitLab instance URL")
+    parser.add_argument(
+        "--oscrc", type=str, help="the OSC configuration with the OBS credentials"
+    )
+    parser.add_argument(
+        "--testing-pipeline-url",
+        type=str,
+        help="The URL of the pipeline running the test",
+    )
+    parser.add_argument(
+        "--ci-config-path",
+        type=str,
+        required=True,
+        help="The path to the CI config to test",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(level=args.loglevel or logging.INFO)
+
+    t = GitLabToOBSTester(
+        reference=args.reference_repository,
+        scratch=args.scratch_repository,
+        testing_pipeline_url=args.testing_pipeline_url,
+    )
+    t.connect(args.gitlab_instance, args.gitlab_server_url, args.gitlab_auth_token, args.oscrc)
+    t.prepare_scratch()
+    t.reset_scratch()
+    t.obs_prepare_work_areas()
+    t.point_gitlab_ci_here(args.ci_config_path)
+    t.test_nonrelease_mr()
+    release_commit = t.test_release_commit()
+    t.test_release_mr_to_frozen_branch(release_commit)
+    t.test_release_mr_to_default_branch(release_commit)
+    logging.info("Test completed successfully ✨")