Add initial version of ci-flatdeb-builder

https://phabricator.apertis.org/T8509

Signed-off-by: Ryan Gonzalez ryan.gonzalez@collabora.com

---

In draft status because this is still using my personal versions of the test repositories, pending the merges of tests/ci-flatdeb-builder-runtimes!1 (merged) and tests/ci-flatdeb-builder-apps!1 (merged). I would also ideally like some feedback on the configuration outlined in docs/apertis-website!476 (merged) first.

tests/conftest.py

+from dataclasses import dataclass
+from pathlib import Path
+
+import enum
+import logging
+import re
+import subprocess
+import sys
+
+from approvaltests.reporters.default_reporter_factory import (  # type: ignore
+    set_default_reporter,
+)
+from approvaltests.reporters.python_native_reporter import (  # type: ignore
+    PythonNativeReporter,
+)
+
+import gitlab
+import pytest
+import yarl
+
+sys.path.append(str(Path(__file__).absolute().parent.parent))
+
+
+STABLE_RELEASE_BRANCH = 'apertis/v2022'
+MAIN_BRANCH = 'main'
+TEST_BRANCH = 'wip/test/fake'
+
+FLATPAK_PRIVATE_KEY = '2b19zVhdsWKxoUOGP8OTBwQGAGSshWJCpage7Ov+tUcoO6slNxwsBDgCq1IBTyDto9XbmLqgSzLVNLqCHgt2UQ=='
+FLATPAK_PUBLIC_KEY = 'KDurJTccLAQ4AqtSAU8g7aPV25i6oEsy1TS6gh4LdlE='
+
+
+@pytest.fixture(scope='session', autouse=True)
+def set_default_reporter_for_all_tests():
+    set_default_reporter(PythonNativeReporter())
+
+
+class TestRepoKind(enum.Enum):
+    RUNTIMES = enum.auto()
+    APPS = enum.auto()
+
+    @property
+    def has_flatpakrepo(self):
+        return self == TestRepoKind.RUNTIMES
+
+
+class TestRepoBranch(enum.Enum):
+    MAIN = enum.auto()
+    TEST = enum.auto()
+
+    @property
+    def versioned_git_branch(self):
+        return {
+            TestRepoBranch.MAIN: STABLE_RELEASE_BRANCH,
+            TestRepoBranch.TEST: TEST_BRANCH,
+        }[self]
+
+    @property
+    def main_only_git_branch(self):
+        return {
+            TestRepoBranch.MAIN: MAIN_BRANCH,
+            TestRepoBranch.TEST: TEST_BRANCH,
+        }[self]
+
+
+@dataclass(frozen=True)
+class TestRepo:
+    project_name: str
+    kind: TestRepoKind
+    branch: TestRepoBranch
+
+    base_upload_root: Path
+
+    base_url: yarl.URL
+    repo_url: yarl.URL
+
+    @staticmethod
+    def create(
+        *,
+        kind: TestRepoKind,
+        project_name: str,
+        branch: TestRepoBranch,
+        upload_root: Path,
+        remote_url: yarl.URL,
+    ) -> 'TestRepo':
+        suffix = f'flatpak-{kind.name.lower()}'
+        base_url = remote_url / suffix
+        base_upload_root = upload_root / suffix
+
+        return TestRepo(
+            kind=kind,
+            project_name=project_name,
+            branch=branch,
+            base_upload_root=base_upload_root,
+            base_url=base_url,
+            repo_url=base_url / branch.name.lower(),
+        )
+
+
+# Make sure pytest doesn't think the Test* classes actually contain test cases.
+TestRepoKind.__test__ = False  # type: ignore
+TestRepoBranch.__test__ = False  # type: ignore
+TestRepo.__test__ = False  # type: ignore
+
+
+def _get_ci_commit_ref_slug():
+    ref_name = subprocess.run(
+        ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
+        stdout=subprocess.PIPE,
+        check=True,
+        text=True,
+    ).stdout.strip()
+    return re.sub('[^0-9A-Za-z]', '-', ref_name[:63])
+
+
+def _get_origin_repo() -> yarl.URL:
+    SSH_RE = re.compile('(?P<user>[^@]+)@(?P<host>[^:]+):(?P<path>.*)')
+
+    url = subprocess.run(
+        ['git', 'config', '--get', 'remote.origin.url'],
+        stdout=subprocess.PIPE,
+        check=True,
+        text=True,
+    ).stdout.strip()
+
+    # Since we're building paths to the GitLab repo interface via this URL, make sure
+    # there's no .git suffix.
+    url = url.removesuffix('.git')
+
+    if match := SSH_RE.match(url):
+        return yarl.URL.build(
+            scheme='https',
+            host=match.group('host'),
+            path='/' + match.group('path'),
+        )
+    else:
+        return yarl.URL(url)
+
+
+def _get_active_git_branch() -> str:
+    return subprocess.run(
+        ['git', 'rev-parse', '--abbrev-ref', 'HEAD'],
+        stdout=subprocess.PIPE,
+        check=True,
+        text=True,
+    ).stdout.strip()
+
+
+def pytest_addoption(parser: pytest.Parser):
+    slug = _get_ci_commit_ref_slug()
+
+    parser.addoption(
+        '--gitlab-instance',
+        help='the GitLab instance from the python-gitlab config file to connect to',
+        default='apertis',
+    )
+
+    parser.addoption(
+        '--gitlab-instance-url',
+        help='the GitLab instance URL to connect to (overrides --gitlab-instance)',
+    )
+    parser.addoption(
+        '--gitlab-instance-token',
+        help='the token to use to connect to --gitlab-instance-url',
+        default='apertis',
+    )
+
+    parser.addoption(
+        '--pipeline-id',
+        help="the GitLab pipeline ID to test; use 'trigger' to trigger a new one",
+    )
+
+    parser.addoption(
+        '--runtimes-test-project',
+        help='the test repository containing flatdeb runtimes & apps',
+        default='tests/ci-flatdeb-builder-runtimes',
+    )
+    parser.addoption(
+        '--apps-test-project',
+        help='the test repository containing normal flatpak-builder apps',
+        default='tests/ci-flatdeb-builder-apps',
+    )
+    parser.addoption(
+        '--ci-config-repo-url',
+        help='the GitLab URL to repository with the CI config to test',
+        default=_get_origin_repo(),
+        type=yarl.URL,
+    )
+    parser.addoption(
+        '--ci-config-repo-use-authentication',
+        help='authenticate access to ci-config-repo (use this if it is private)',
+        action='store_true',
+    )
+    parser.addoption(
+        '--ci-config-repo-revision',
+        help='the revision of ci-config-repo to use',
+        default=_get_active_git_branch(),
+    )
+    parser.addoption(
+        '--remote-url',
+        help='URL where build artifacts will be downloaded from',
+        default=f'https://images.apertis.org/test/ci-flatdeb-builder-{slug}',
+        type=yarl.URL,
+    )
+    parser.addoption(
+        '--upload-root',
+        help='path where build artifacts will be uploaded to on the build server',
+        default=f'/srv/images/test/ci-flatdeb-builder-{slug}',
+        type=Path,
+    )
+
+
+@pytest.fixture(scope='session')
+def pipeline_id(request) -> str:
+    p = request.config.option.pipeline_id
+    assert p is not None, 'Test requires --pipeline-id to be set'
+    assert p == 'trigger' or p.isdigit(), f'Invalid pipeline ID: {p}'
+    return p
+
+
+@pytest.fixture(scope='session')
+def gitlab_instance(request) -> gitlab.Gitlab:
+    if url := request.config.option.gitlab_instance_url:
+        token = request.config.option.gitlab_instance_token
+
+        logging.info(f'connecting to GitLab instance URL {url}')
+        return gitlab.Gitlab(url, private_token=token)
+    else:
+        instance = request.config.option.gitlab_instance
+        logging.info(f'connecting to configured GitLab instance {instance}')
+        return gitlab.Gitlab.from_config(request.config.option.gitlab_instance)
+
+
+@pytest.fixture(scope='session')
+def runtimes_test_project(request) -> str:
+    return request.config.option.runtimes_test_project
+
+
+@pytest.fixture(scope='session')
+def apps_test_project(request) -> str:
+    return request.config.option.apps_test_project
+
+
+@pytest.fixture(scope='session')
+def ci_config_repo_url(request) -> yarl.URL:
+    url: yarl.URL = request.config.option.ci_config_repo_url
+
+    if request.config.option.ci_config_repo_use_authentication:
+        token = request.config.option.gitlab_instance_token
+        url = url.with_user('oauth2').with_password(token)
+
+    return url
+
+
+@pytest.fixture(scope='session')
+def ci_config_repo_revision(request) -> str:
+    return request.config.option.ci_config_repo_revision
+
+
+@pytest.fixture(scope='session')
+def ci_config_file_url(
+    ci_config_repo_url: yarl.URL,
+    ci_config_repo_revision: str,
+) -> yarl.URL:
+    return (
+        ci_config_repo_url
+        / '-'
+        / 'raw'
+        / ci_config_repo_revision
+        / 'ci-flatdeb-builder.yaml'
+    )
+
+
+@pytest.fixture(scope='session')
+def remote_url(request) -> yarl.URL:
+    print(request.config.option.remote_url)
+    return request.config.option.remote_url
+
+
+@pytest.fixture(scope='session')
+def upload_root(request) -> Path:
+    return request.config.option.upload_root
+
+
+@pytest.fixture(
+    scope='session',
+    params=[
+        pytest.param(v, marks=getattr(pytest.mark, f'gitlab_repo_{v.name.lower()}'))
+        for v in TestRepoKind
+    ],
+    ids=[v.name.lower() for v in TestRepoKind],
+)
+def test_repo_kind(request):
+    return request.param
+
+
+@pytest.fixture(
+    scope='session',
+    params=[
+        pytest.param(v, marks=getattr(pytest.mark, f'gitlab_branch_{v.name.lower()}'))
+        for v in TestRepoBranch
+    ],
+    ids=[v.name.lower() for v in TestRepoBranch],
+)
+def test_repo_branch(request):
+    return request.param
+
+
+@pytest.fixture(scope='session')
+def test_repo(
+    test_repo_kind: TestRepoKind,
+    test_repo_branch: TestRepoBranch,
+    upload_root: Path,
+    remote_url: yarl.URL,
+    runtimes_test_project: str,
+    apps_test_project: str,
+):
+    if test_repo_kind == TestRepoKind.RUNTIMES:
+        project_name = runtimes_test_project
+    else:
+        assert test_repo_kind == TestRepoKind.APPS
+        project_name = apps_test_project
+
+    return TestRepo.create(
+        project_name=project_name,
+        kind=test_repo_kind,
+        branch=test_repo_branch,
+        upload_root=upload_root,
+        remote_url=remote_url,
+    )