import-debian-package: Allow creating/pushing to remote repository

To import a new package from Debian the current process required several
manual steps. This commit adds the --push-remote option to allow
creating and pushing the imported package to apertis remote gitlab
instance.

The following steps are now automatized:
* package folder is created and import process is performed in there.
* remote repo is created if it doesn't exist.
* remote branches and tags are pushed.
* gitlab settings are applied to remote repo.
* CI pipeline is triggered on all downstream branches.

Signed-off-by: Ariel D'Alessandro <ariel.dalessandro@collabora.com>

debian/apertis/local-gitlab-ci.yml

@@ -24,6 +24,7 @@ test-import-debian-package:
     - apt-get install -y --no-install-recommends
       patchutils dctrl-tools devscripts
       pristine-lfs python3-debian python3-sh
+      python3-gitlab gitlab-rulez
       git-buildpackage dgit
     - tests/import-debian-package/run-tests.sh
   needs:

debian/control

@@ -32,6 +32,7 @@ Depends:
  python3-yaml,
  python3-paramiko,
  python3-gi,
+ python3-gitlab,
  systemd-container
 Suggests:
  ribchester (>= 0.1612.7)

tools/import-debian-package

@@ -18,6 +18,7 @@ import logging
 import argparse
 import sh
 import json
+import gitlab
 from tempfile import TemporaryDirectory
 from sh.contrib import git
 from sh import dget, debsnap
@@ -35,11 +36,35 @@ from gbp.deb.git import DebianGitRepository
 import gbp.log
 from typing import Mapping, Optional
 
+gitlab_rulez = sh.Command("gitlab-rulez")
 pristine_lfs = sh.Command("pristine-lfs")
 
+APERTIS_GITLAB_DOMAIN = 'gitlab.apertis.org'
+APERTIS_GITLAB_GROUP = 'pkg'
+APERTIS_INFRASTRUCTURE_REPO = f'git@{APERTIS_GITLAB_DOMAIN}:infrastructure/apertis-infrastructure.git'
 APERTIS_MAINTAINERS_NAME = 'Apertis package maintainers'
 APERTIS_MAINTAINERS_EMAIL = 'packagers@lists.apertis.org'
 
+class GitlabWrapper:
+    def __init__(self):
+        self.gl = None
+
+    def connect(self, gitlab_instance, gitlab_server_url, gitlab_api_token):
+        if gitlab_server_url:
+            self.gl = gitlab.Gitlab(gitlab_server_url, private_token=gitlab_api_token)
+        else:
+            self.gl = gitlab.Gitlab.from_config(gitlab_instance)
+            self.gl.auth()
+
+    def get_project(self, name):
+        return self.gl.projects.get(f'{APERTIS_GITLAB_GROUP}/{name}')
+
+    def get_group(self, name):
+        return self.gl.groups.list(search=name)[0].id
+
+    def create_project(self, name):
+        return self.gl.projects.create({'name': name, 'namespace_id': self.get_group(APERTIS_GITLAB_GROUP), 'visibility': 'public'})
+
 def gbp_import_dsc(*args, **kwargs):
     logging.debug('gbp import-dsc %r', args)
     return gbp_import_dsc_orig(*args, **kwargs)
@@ -217,9 +242,23 @@ def fetch(package: str, version: str, url: str, downloaddir: str, cached_dsc: st
     downloaded = Path(downloaddir) / dsc
     return downloaded
 
+def apply_gitlab_rulez(gitlab_instance, gitlab_server_url, gitlab_api_token, package):
+    with TemporaryDirectory() as infrastructuredir:
+        git.clone(APERTIS_INFRASTRUCTURE_REPO, infrastructuredir)
+        args = []
+        if gitlab_instance:
+            args += ['--gitlab-instance', gitlab_instance]
+        if gitlab_server_url:
+            args += ['--gitlab-server-url', gitlab_server_url]
+        if gitlab_api_token:
+            args += ['--gitlab-api-token', gitlab_api_token]
+        args += ['apply', f'{infrastructuredir}/gitlab-scripts/rulez.yaml', '--filter', f'{APERTIS_GITLAB_GROUP}/{package}']
+        gitlab_rulez(args)
+
 def do_import(args):
     os.environ['GBP_CONF_FILES'] = '/dev/null'
 
+    os.chdir(ensure_dir_or_none(args.package))
     if not Path('.git').is_dir():
         git.init()
     configure_user_email(args.maint_name, args.maint_email)
@@ -319,6 +358,34 @@ def do_import(args):
         for branch in args.downstreams:
             force_branch(branch, 'HEAD')
 
+    if args.push_remote:
+        r = GitlabWrapper()
+        r.connect(args.gitlab_instance, args.gitlab_server_url, args.gitlab_api_token)
+
+        try:
+            project = r.get_project(args.package)
+            logging.info('Using existing remote repository')
+        except:
+            logging.info('Creating new remote repository')
+            project = r.create_project(args.package)
+
+        try:
+            remote_url = git('remote', 'get-url', 'origin')
+            logging.warning(f'Remote was already set to: {remote_url}')
+            logging.warning(f'Will overwrite with remote repository url')
+            git('remote', 'set-url', 'origin', project.ssh_url_to_repo)
+        except:
+            git('remote', 'add', 'origin', project.ssh_url_to_repo)
+
+        logging.info('Pushing remote branches and tags')
+        git.push('--all', '--follow-tags', 'origin')
+
+        logging.info('Applying Gitlab rules')
+        apply_gitlab_rulez(args.gitlab_instance, args.gitlab_server_url, args.gitlab_api_token, args.package)
+
+        logging.info('Triggering CI pipelines on downstream branches')
+        for branch in [downstream] + args.downstreams:
+            project.pipelines.create({'ref': branch})
 
 def main(argv):
     prog = os.path.basename(argv[0])
@@ -339,6 +406,10 @@ def main(argv):
     parser.add_argument('--component', dest='component', type=str, default="development", metavar='REPO_COMPONENT', help='specify the repository component. Eg: target')
     parser.add_argument('--maintainer-name', dest='maint_name', type=str, default=APERTIS_MAINTAINERS_NAME, metavar='NAME', help='the changelog author name')
     parser.add_argument('--maintainer-email', dest='maint_email', type=str, default=APERTIS_MAINTAINERS_EMAIL, metavar='EMAIL', help='the changelog author email')
+    parser.add_argument("--gitlab-instance", type=str, default="apertis", help="get connection parameters from this configured instance")
+    parser.add_argument("--gitlab-api-token", type=str, help="the GitLab API token")
+    parser.add_argument("--gitlab-server-url", type=str, help="the GitLab instance URL")
+    parser.add_argument('--push-remote', action='store_true', default=False, help='push package to remote repository')
 
     args = parser.parse_args()
     if isinstance(args.downstreams, str):