packages-sync: Checks that package versions are synchronized

Look at package versions across Apertis releases to detect if we've missed
an update.

Signed-off-by: Emanuele Aina <emanuele.aina@collabora.com>

packages-sync/Dockerfile

+FROM debian:buster-slim
+
+RUN apt-get update && apt-get install -y \
+    python3 \
+    python3-debian \
+    python3-jinja2 \
+    python3-requests \
+    python3-requests-cache \
+    --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/*

packages-sync/Jenkinsfile

+pipeline {
+  agent {
+    dockerfile {
+      filename 'packages-sync/Dockerfile'
+    }
+  }
+  environment {
+    RELEASE = "v2019"
+    REFERENCES = "v2019dev0 v2020dev0"
+  }
+  stages {
+    stage('Test') {
+      steps {
+        sh '''
+          REFS=$(echo $REFERENCES | sed -e 's/\\([^ ]\\+\\)/--reference \\1/g')
+          ./packages-sync/packages-sync --release $RELEASE $REFS --junit junit.xml
+        '''
+      }
+    }
+  }
+  post {
+    success {
+      junit 'junit.xml'
+    }
+  }
+}

packages-sync/packages-sync

+#!/usr/bin/env python3
+
+import argparse
+import dataclasses
+import gzip
+import io
+import textwrap
+import sys
+
+import debian.deb822
+import debian.debian_support
+import jinja2
+import requests
+import requests_cache
+
+@dataclasses.dataclass
+class Package:
+  release: str
+  name: str
+  version: debian.debian_support.Version
+  others: list = dataclasses.field(default_factory=list)
+
+def source_packages_by_component(release, component):
+  url = f'https://repositories.apertis.org/apertis/dists/{release}/{component}/source/Sources.gz'
+  r = requests.get(url, stream=True)
+  fields = ('Package', 'Version')
+  with gzip.open(io.BytesIO(r.content)) as gz:
+    yield from debian.deb822.Packages.iter_paragraphs(gz, use_apt_pkg=False, fields=fields)
+
+def source_packages_by_release(release):
+  components = [
+    'target',
+    'development',
+    'sdk',
+    'hmi',
+    ]
+  for component in components:
+    yield from source_packages_by_component(release, component)
+
+def get_packages_dict(release):
+  return {pkg['Package']: pkg.get_version() for pkg in source_packages_by_release(release)}
+
+def get_packages(release, references):
+  packages = {}
+  packages[release] = get_packages_dict(release)
+  for ref in references:
+    packages[ref] = get_packages_dict(ref)
+  return packages
+
+def compare(packages, release, references):
+  checks = []
+  for name, version in packages[release].items():
+    others = []
+    for ref in references:
+      other = packages[ref].get(name)
+      if version < other:
+        others.append(Package(ref, name, other))
+    checks.append(Package(release, name, version, others))
+  return checks
+
+def render_junit(project, release, packages):
+  junit_template = textwrap.dedent("""\
+    <?xml version="1.0" encoding="UTF-8"?>
+    <testsuites tests="{{packages|length}}" failures="{{packages|selectattr('others')|list|length}}">
+      <testsuite name="{{project}}:{{release}}">
+        {%- for p in packages %}
+        <testcase name="{{p.name}}">
+          {%- if p.others %}
+          <failure type="failed">
+            {%- for o in p.others %}
+            {{p.release}}/{{p.name}}_{{p.version}} &lt; {{o.release}}/{{o.name}}_{{o.version}}
+            {%- endfor %}
+          </failure>
+          {%- endif %}
+        </testcase>
+        {%- endfor %}
+      </testsuite>
+    </testsuites>
+  """)
+  environment = jinja2.Environment(autoescape=True)
+  template = environment.from_string(junit_template)
+  result = template.render(
+    project=project,
+    release=release,
+    packages=packages,
+  )
+  return result
+
+def main(args):
+  parser = argparse.ArgumentParser(description='Checks that no package in the reference releases in newer than the one in RELEASE')
+  parser.add_argument('--ignore', type=argparse.FileType('r'), help='a file containing the list of packages to ignore')
+  parser.add_argument('--release', required=True, type=str, metavar='RELEASE', help='the release to check')
+  parser.add_argument('--reference', required=True, type=str, action='append', help='a reference release')
+  parser.add_argument('--cache', type=str, help='the path where to store the local HTTP cache')
+  parser.add_argument('--junit', type=argparse.FileType('w'), help='file to store results in JUnit format')
+
+  args = parser.parse_args(args)
+
+  if args.cache:
+    requests_cache.install_cache(cache_name=args.cache)
+
+  packages = get_packages(args.release, args.reference)
+  checks = compare(packages, args.release, args.reference)
+  for p in checks:
+    for o in p.others:
+      print(f'{p.release}/{p.name}_{p.version} < {o.release}/{o.name}_{o.version}')
+  if args.junit:
+    args.junit.write(render_junit('apertis', args.release, checks))
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))