tests: Test the ci-package-builder pipeline

Set up a test repository and run the ci-package-builder pipeline on it
by committing new changes, creating merge requests and monitoring the
resulting pipelines.

The test currently checks:
* submitting non-release changes and landing them
* submitting release commits and landing them
* blocking commits to frozen stable branches

This pipeline needs some extra setup:
* a tests/dash> repository, forked from pkg/target/dash>, where we
  force-push changes, create merge requests and monitor pipelines
* the `GITLAB_CI_USER`, `GITLAB_CI_PASSWORD`, `OSC_USERNAME` and
  `OSC_PASSWORD` CI variables to be set on tests/dash>, matching what it
  is used on the pkg/ projects
* the `TEST_GITLAB_AUTH_TOKEN` CI variable to be set on this repository
  to access the GitLab APIs used to issue MRs and monitor pipelines, and
  to push changes via git to the tests/dash> repository
* the `TEST_OSCRC` CI variable to be set on this repository to prepare
  the branch projects where the upload jobs will be tested

Signed-off-by: Emanuele Aina <emanuele.aina@collabora.com>

.gitlab-ci.yml

+run-dash-packaging-pipelines:
+  stage: test
+  image: debian:bullseye
+  resource_group: tests/dash # run a job at a time against the test repository
+  variables:
+    SCRATCH_REPO: em/dash # FIXME: move to test/dash
+    REFERENCE_REPO: pkg/target/dash
+  before_script:
+    - apt update && apt install --no-install-recommends -y
+        devscripts
+        osc
+        python3-git
+        python3-gitlab
+        python3-m2crypto
+  script:
+    - ./tests/test_packaging_pipelines.py
+        --oscrc "${TEST_OSCRC}"
+        --gitlab-auth-token "${TEST_GITLAB_AUTH_TOKEN}"
+        --gitlab-server-url "${CI_SERVER_URL}"
+        --scratch-repository "${SCRATCH_REPO}"
+        --reference-repository "${REFERENCE_REPO}"
+        --ci-config-path "ci-package-builder.yml@${CI_PROJECT_PATH}"
+        --debug
+  rules:
+    - if: $CI_MERGE_REQUEST_ID
+      when: never
+    - if: $CI_COMMIT_BRANCH == "master"
+      when: on_success
+    - when: on_success # never
+
+

tests/test_packaging_pipelines.py

+#!/usr/bin/env python3
+
+import argparse
+import logging
+import pathlib
+import re
+import subprocess
+import time
+import urllib.parse
+
+import git
+import gitlab
+import osc.conf
+import osc.core
+
+SCRATCH_REPO_DIR = "/tmp/scratchrepo"
+TEST_BRANCH = "wip/test/fake"
+WAIT_AFTER_PUSH = 5
+
+
+def _debian_mangle_version_to_tag(v):
+    # see https://dep-team.pages.debian.net/deps/dep14/
+    v = v.replace(":", "%")
+    v = v.replace("~", "_")
+    v = re.sub("\.(?=\.|$|lock$)", ".#", v)
+    return v
+
+
+def _get_branch_prefix(project):
+    sep = "/"
+    prefix = project.default_branch.split(sep)[0]
+    return prefix + sep
+
+
+def _find_stable_release_branch(project, prefix):
+    branches = (b.name for b in project.branches.list(all=True))
+    releases = (b for b in branches if b.startswith(prefix))
+    stable = (b for b in releases if b[-4:-1] != "dev" and b[-3:] != "pre")
+    latest = sorted(stable, reverse=True)[0]
+    return latest
+
+
+def _monitor_pipeline_for_completion(pipeline):
+    logging.info(f"monitoring pipeline {pipeline.web_url}")
+    while pipeline.status in ("running", "pending", "created"):
+        logging.debug(f"pipeline {pipeline.web_url} status is {pipeline.status}")
+        pipeline.refresh()
+        time.sleep(5)
+
+
+def _commit_and_push(gl, repo, filename, msg):
+    repo.index.add(filename)
+    actor = git.Actor(gl.user.name, gl.user.email)
+    commit = repo.index.commit(msg, author=actor, committer=actor)
+    logging.info(f"committed {commit.hexsha} to {TEST_BRANCH}")
+    pushes = repo.remotes.origin.push(TEST_BRANCH, force=True, verbose=True)
+    assert not any(p.flags & git.remote.PushInfo.ERROR for p in pushes)
+    time.sleep(
+        WAIT_AFTER_PUSH
+    )  # give GitLab some time to realize we pushed some changes
+    return commit
+
+
+class GitLabToOBSTester:
+    def __init__(self, reference, scratch, testing_pipeline_url):
+        self.apiurl = None
+        self.gl = None
+        self.reference = reference
+        self.scratch = scratch
+        self.scratch_git = None
+        self.scratch_gitlab = None
+        self.stable_branch = None
+        self.testing_pipeline_url = testing_pipeline_url
+
+    def connect(self, gitlab_instance, gitlab_server_url, gitlab_auth_token, oscrc):
+        if gitlab_server_url:
+            logging.info(f'Connecting to the "{gitlab_server_url}" instance')
+            self.gl = gitlab.Gitlab(gitlab_server_url, private_token=gitlab_auth_token)
+        else:
+            logging.info(f'Connecting to the "{gitlab_instance}" configured instance')
+            self.gl = gitlab.Gitlab.from_config(gitlab_instance)
+        self.gl.auth()
+        osc.conf.get_config(override_conffile=oscrc)
+        self.apiurl = osc.conf.config["apiurl"]
+
+    def _scratch_set_properties(self):
+        logging.info(f"setting up properties on {self.scratch}, temporarily disable CI")
+        s = self.gl.projects.get(self.scratch)
+        self.scratch_gitlab = s
+        try:
+            s.branches.delete(TEST_BRANCH)
+        except gitlab.GitlabDeleteError:
+            pass
+        s.merge_requests_access_level = "enabled"
+        s.merge_method = "ff"
+        s.builds_access_level = "disabled"
+        s.lfs_enabled = True
+        s.save()
+
+    def _scratch_clone(self):
+        url = urllib.parse.urlparse(self.scratch_gitlab.http_url_to_repo)
+        password = urllib.parse.quote(self.gl.private_token, safe="")
+        netloc = f"oauth2:{password}@{url.netloc}"
+        url = url._replace(netloc=netloc)
+        logging.info(f"clone {url.geturl()} to {SCRATCH_REPO_DIR}")
+        self.scratch_git = git.Repo.clone_from(url.geturl(), SCRATCH_REPO_DIR)
+
+    def prepare_scratch(self):
+        self._scratch_set_properties()
+        self._scratch_clone()
+
+    def _scratch_set_variables(self):
+        obs_project_prefix = self._get_obs_test_project_prefix()
+        logging.info(f"force OBS projects under {obs_project_prefix}")
+        varname = "OBS_PREFIX"
+        vardata = {"key": varname, "value": obs_project_prefix}
+        try:
+            self.scratch_gitlab.variables.update(varname, vardata)
+        except gitlab.GitlabUpdateError:
+            self.scratch_gitlab.variables.create(vardata)
+
+    def _scratch_cleanup_tags(self):
+        scratch = self.scratch_git
+        old_tags = list(scratch.tags)
+        logging.debug(f"cleaning up the remote tags {old_tags}")
+        if old_tags:
+            pushes = scratch.remotes.origin.push(old_tags, delete=True, verbose=True)
+            assert not any(p.flags & git.remote.PushInfo.ERROR for p in pushes)
+            scratch.delete_tag(old_tags)
+
+    def _scratch_synchronize_branches(self, reference):
+        scratch = self.scratch_git
+        default = reference.default_branch
+        logging.debug("synchronizing branches")
+        prefix = _get_branch_prefix(reference)
+        self.stable_branch = _find_stable_release_branch(reference, prefix)
+        branches = [default, self.stable_branch, "pristine-lfs", "pristine-lfs-source"]
+        logging.info(f"synchronizing the {branches} branches")
+        for branchname in branches:
+            scratch.create_head(
+                branchname,
+                commit=scratch.remotes.reference.refs[branchname],
+                force=True,
+            )
+        logging.debug(
+            f"pushing {branches} to {self.scratch_gitlab.path_with_namespace}"
+        )
+        pushes = scratch.remotes.origin.push(
+            branches, force=True, verbose=True, follow_tags=True
+        )
+        assert not any(p.flags & git.remote.PushInfo.ERROR for p in pushes)
+
+    def reset_scratch(self):
+        scratch = self.scratch_git
+        logging.info(
+            f"resetting {self.scratch_gitlab.path_with_namespace} to {self.reference}"
+        )
+        self._scratch_cleanup_tags()
+        reference = self.gl.projects.get(self.reference)
+        default = reference.default_branch
+        url = reference.http_url_to_repo
+        scratch.create_remote("reference", url)
+        scratch.remotes.reference.fetch()
+        head = scratch.remotes.reference.refs[default]
+        logging.debug(f"switching to the {TEST_BRANCH} branch as {head.commit.hexsha}")
+        scratch.create_head(TEST_BRANCH, commit=head.commit).checkout()
+        self._scratch_synchronize_branches(reference)
+        logging.debug(
+            f"setting the default branch on {self.scratch_gitlab.path_with_namespace} to {default} and re-enable CI"
+        )
+        self.scratch_gitlab.default_branch = default
+        self.scratch_gitlab.builds_access_level = "enabled"
+        self.scratch_gitlab.save()
+        self._scratch_set_variables()
+
+    def _get_obs_reference_project_name(self):
+        root = self.scratch_gitlab.default_branch.replace("/", ":")
+        component = "development"
+        filepath = pathlib.Path(SCRATCH_REPO_DIR, "debian/apertis/component")
+        if filepath.exists():
+            component = open(filepath).read().strip()
+        project = f"{root}:{component}"
+        return project
+
+    def _get_obs_test_project_prefix(self):
+        username = osc.conf.config["api_host_options"][self.apiurl]["user"]
+        test_project = f"home:{username}:branches:test:"
+        return test_project
+
+    def _get_obs_test_project_name(self):
+        prefix = self._get_obs_test_project_prefix()
+        reference_project = self._get_obs_reference_project_name()
+        test_project = prefix + reference_project
+        return test_project
+
+    def obs_prepare_work_areas(self):
+        logging.info("preparing work areas on OBS")
+        package = self.reference.split("/")[-1]
+        src_project = self._get_obs_reference_project_name()
+        target_project = self._get_obs_test_project_name()
+        msg = "Branch for testing the GitLab-to-OBS pipeline"
+        for suffix in ("", ":snapshots"):
+            logging.info(
+                f"branching {package} from {src_project} to {target_project + suffix}"
+            )
+            osc.core.branch_pkg(
+                self.apiurl,
+                src_project,
+                package,
+                target_project=target_project + suffix,
+                msg=msg,
+                force=True,
+            )
+
+    def point_gitlab_ci_here(self, ci_config_path):
+        logging.info(f"pointing to the CI definition at {ci_config_path}")
+        self.scratch_gitlab.ci_config_path = ci_config_path
+        self.scratch_gitlab.save()
+
+    def test_nonrelease_mr(self):
+        logging.info(f"testing an unreleased version on {TEST_BRANCH}")
+        assert self.scratch_git.active_branch.name == TEST_BRANCH
+        subprocess.run(
+            ["dch", "-i", "Fake changes while testing the GitLab-to-OBS pipeline"],
+            check=True,
+            cwd=SCRATCH_REPO_DIR,
+        )
+        msg = f"Test unreleased changes\n\nPipeline: {self.testing_pipeline_url}"
+        commit = _commit_and_push(self.gl, self.scratch_git, "debian/changelog", msg)
+        pipeline = self.scratch_gitlab.pipelines.list(
+            ref=TEST_BRANCH, sha=commit.hexsha
+        )[0]
+        _monitor_pipeline_for_completion(pipeline)
+        assert (
+            pipeline.status == "success"
+        ), f"submitted non-release pipeline {pipeline.web_url} didn't succeed: {pipeline.status}"
+        job_names = [j.name for j in pipeline.jobs.list()]
+        assert job_names == [
+            "build-source"
+        ], f"submitted non-release pipeline on unmerged changes expected different jobs: {job_names}"
+        target = self.scratch_gitlab.default_branch
+        mr = self.scratch_gitlab.mergerequests.create(
+            dict(
+                source_branch=TEST_BRANCH,
+                target_branch=target,
+                title="Test non-release",
+            )
+        )
+        logging.info(f"created non-release MR {mr.web_url}")
+        pipelines = [p for p in mr.pipelines() if p["sha"] == commit.hexsha]
+        assert (
+            len(pipelines) == 1
+        ), f"non-release MR has unexpected pipelines: {' '.join(p['web_url'] for p in pipelines)}"
+        assert mr.pipelines()[0]["id"] == pipeline.id
+        logging.info(f"landing {mr.web_url}")
+        time.sleep(WAIT_AFTER_PUSH)
+        mr.merge()
+        time.sleep(WAIT_AFTER_PUSH)
+        pipeline = self.scratch_gitlab.pipelines.list(
+            ref=self.scratch_gitlab.default_branch, sha=commit.hexsha
+        )[0]
+        _monitor_pipeline_for_completion(pipeline)
+        assert (
+            pipeline.status == "success"
+        ), f"landed non-release pipeline {pipeline.web_url} didn't succeed: {pipeline.status}"
+        job_names = [j.name for j in pipeline.jobs.list()]
+        assert job_names == [
+            "build-source",
+            "tag-release",
+            "upload",
+        ], f"landed non-release pipeline expected different jobs: {job_names}"
+        logging.info("Non-release MR landed successfully ✅")
+
+    def test_release_commit(self):
+        logging.info("submitting a release commit")
+        subprocess.run(["dch", "-r", "apertis"], check=True, cwd=SCRATCH_REPO_DIR)
+        msg = f"Test release commit\n\nPipeline: {self.testing_pipeline_url}"
+        commit = _commit_and_push(self.gl, self.scratch_git, "debian/changelog", msg)
+        pipeline = self.scratch_gitlab.pipelines.list(
+            ref=TEST_BRANCH, sha=commit.hexsha
+        )[0]
+        _monitor_pipeline_for_completion(pipeline)
+        assert (
+            pipeline.status == "success"
+        ), f"submitted release pipeline {pipeline.web_url} didn't succeed: '{pipeline.status}'"
+        job_names = [j.name for j in pipeline.jobs.list()]
+        assert job_names == [
+            "build-source"
+        ], f"submitted release pipeline on unmerged changes expected different jobs: {job_names}"
+        return commit
+
+    def test_release_mr_to_frozen_branch(self, commit):
+        logging.info("testing MR to frozen branch")
+        target = self.stable_branch
+        mr = self.scratch_gitlab.mergerequests.create(
+            dict(
+                source_branch=TEST_BRANCH,
+                target_branch=target,
+                title="Test release on frozen branch",
+            )
+        )
+        logging.info(f"created release MR {mr.web_url} to frozen branch {target}")
+        time.sleep(WAIT_AFTER_PUSH)
+        pipelines = {
+            p["ref"]: self.scratch_gitlab.pipelines.get(p["id"])
+            for p in mr.pipelines()
+            if p["sha"] == commit.hexsha
+        }
+        assert (
+            len(pipelines) == 2
+        ), f"unexpected pipelines: {p['web_url'] for p in pipelines}"
+        branch_pipeline = pipelines.pop(TEST_BRANCH)
+        mr_pipeline = next(iter(pipelines.values()))
+        _monitor_pipeline_for_completion(branch_pipeline)
+        _monitor_pipeline_for_completion(mr_pipeline)
+        assert (
+            branch_pipeline.status == "success"
+        ), f"release branch pipeline {branch_pipeline.web_url} to frozen branch didn't succeed: '{branch_pipeline.status}'"
+        assert (
+            mr_pipeline.status == "failed"
+        ), f"release MR pipeline {mr_pipeline.web_url} to frozen branch should have failed: '{mr_pipeline.status}'"
+        job_names = [j.name for j in mr_pipeline.jobs.list()]
+        assert job_names == [
+            "freeze-stable-branches"
+        ], f"submitted non-release pipeline expected different jobs: {job_names}"
+        mr.state_event = "close"
+        mr.save()
+        logging.info("MR on frozen branch blocked successfully ✅")
+
+    def test_release_mr_to_default_branch(self, commit):
+        logging.info("testing MR to default branch")
+        target = self.scratch_gitlab.default_branch
+        mr = self.scratch_gitlab.mergerequests.create(
+            dict(source_branch=TEST_BRANCH, target_branch=target, title="Test release")
+        )
+        logging.info(f"created release MR {mr.web_url} to default branch {target}")
+        pipelines = [p for p in mr.pipelines() if p["sha"] == commit.hexsha]
+        assert len(pipelines) == 1
+        assert pipelines[0]["ref"] == TEST_BRANCH
+        logging.info(f"landing {mr.web_url}")
+        time.sleep(WAIT_AFTER_PUSH)
+        mr.merge()
+        time.sleep(WAIT_AFTER_PUSH)
+        pipeline = self.scratch_gitlab.pipelines.list(
+            ref=self.scratch_gitlab.default_branch, sha=commit.hexsha
+        )[0]
+        _monitor_pipeline_for_completion(pipeline)
+        assert (
+            pipeline.status == "success"
+        ), f"landed release pipeline {pipeline.web_url} didn't succeed: '{pipeline.status}'"
+        job_names = [j.name for j in pipeline.jobs.list()]
+        assert job_names == [
+            "build-source",
+            "tag-release",
+            "upload",
+        ], f"landed release pipeline on merged changes expected different jobs: {job_names}"
+        logging.info("Release MR landed successfully ✅")
+
+    def test_release_artifacts(self, commit):
+        cmd = subprocess.run(
+            ["dpkg-parsechangelog", "-SVersion"],
+            capture_output=True,
+            check=True,
+            cwd=SCRATCH_REPO_DIR,
+        )
+        version = cmd.stdout.decode().strip()
+        mangledversion = _debian_mangle_version_to_tag(version)
+        prefix = self.scratch_gitlab.default_branch.split("/")[0]
+        tagname = f"{prefix}/{mangledversion}"
+        self.scratch_git.remotes.origin.fetch(tags=True)
+        tag = self.scratch_git.tags[tagname]
+        assert (
+            tag.commit == commit
+        ), f"tag {tagname} points to {tag.commit}, expected {commit}"
+        project = self._get_obs_test_project_name()
+        package = self.reference.split("/")[-1]
+        files = osc.core.meta_get_filelist(self.apiurl, project, package)
+        dscs = [f for f in files if f.endswith(".dsc")]
+        assert len(dscs) == 1, "expected a single .dsc, got {dscs}"
+        dsc = dscs[0]
+        expected = f"{package}_{version}.dsc"
+        assert dsc == expected, "expected '{expected}' on OBS, got '{dsc}'"
+        logging.info(f"Found {tagname} and {dsc} succesfully ✅")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Test the GitLab-to-OBS pipeline")
+    parser.add_argument(
+        "--debug",
+        action="store_const",
+        dest="loglevel",
+        const=logging.DEBUG,
+        help="print debug information",
+    )
+    parser.add_argument(
+        "--scratch-repository",
+        type=str,
+        required=True,
+        help="the repository on which the pipeline is tested",
+    )
+    parser.add_argument(
+        "--reference-repository",
+        type=str,
+        required=True,
+        help="the scratch repository will be reset to the reference-repository contents",
+    )
+    parser.add_argument(
+        "--gitlab-instance",
+        type=str,
+        default="apertis",
+        help="get connection parameters from this configured instance",
+    )
+    parser.add_argument(
+        "--gitlab-auth-token", type=str, help="the GitLab authentication token"
+    )
+    parser.add_argument("--gitlab-server-url", type=str, help="the GitLab instance URL")
+    parser.add_argument(
+        "--oscrc", type=str, help="the OSC configuration with the OBS credentials"
+    )
+    parser.add_argument(
+        "--testing-pipeline-url",
+        type=str,
+        help="The URL of the pipeline running the test",
+    )
+    parser.add_argument(
+        "--ci-config-path",
+        type=str,
+        required=True,
+        help="The path to the CI config to test",
+    )
+    args = parser.parse_args()
+
+    logging.basicConfig(level=args.loglevel or logging.INFO)
+
+    t = GitLabToOBSTester(
+        reference=args.reference_repository,
+        scratch=args.scratch_repository,
+        testing_pipeline_url=args.testing_pipeline_url,
+    )
+    t.connect(
+        args.gitlab_instance, args.gitlab_server_url, args.gitlab_auth_token, args.oscrc
+    )
+    t.prepare_scratch()
+    t.reset_scratch()
+    t.obs_prepare_work_areas()
+    t.point_gitlab_ci_here(args.ci_config_path)
+    t.test_nonrelease_mr()
+    release_commit = t.test_release_commit()
+    t.test_release_mr_to_frozen_branch(release_commit)
+    t.test_release_mr_to_default_branch(release_commit)
+    t.test_release_artifacts(release_commit)
+    logging.info("Test completed successfully ✨")