Commit d862282b authored by Emanuele Aina's avatar Emanuele Aina
Browse files

packaging-check-installability: Run dose-distcheck on APT repositories



Check the dependencies of the packages published in the APT repositories
to verify there's no dependency issue preventing them to be installed.

This adds a separate report page where the installbility errors are
grouped by the actual package (usually a transitive dependency) which is
actually uninstallable. This shows which packages are broken for the
same reason, and points to the package to fix if the broken packages
should be installable.

Ignored errors are also shown as notes in the installability report,
providing a way to review the effects of the ignore rules.
Signed-off-by: Emanuele Aina's avatarEmanuele Aina <emanuele.aina@collabora.com>
parent 6de44d63
Pipeline #176744 passed with stages
in 57 minutes and 26 seconds
......@@ -85,6 +85,25 @@ packaging-data-fetch-obs:
paths:
- packaging-data-obs.yaml
packaging-check-installability:
stage: fetch
tags:
- lightweight
before_script:
- apt update && apt install -y --no-install-recommends
ca-certificates
dose-distcheck
python3-coloredlogs
python3-debian
python3-requests
python3-yaml
script:
- ./bin/packaging-check-installability
--yaml packaging-check-installability.yaml
artifacts:
paths:
- packaging-check-installability.yaml
packaging-check-invariants:
stage: check
tags:
......@@ -99,6 +118,7 @@ packaging-check-invariants:
--input packaging-data-downstream.yaml
--input packaging-data-upstream.yaml
--input packaging-data-obs.yaml
--input packaging-check-installability.yaml
--output packaging-data.yaml
- ./bin/packaging-check-invariants
--projects packaging-data.yaml
......
......@@ -10,27 +10,70 @@ import jinja2
import yaml
def _is_installability_error(error):
return "missing" in error.get("details", [{}])[0]
def preprocess(data):
packages = data["packages"].values()
installability_errors_count = sum(
any(map(_is_installability_error, p.get("errors", []))) for p in packages
)
installability_notes_count = sum(
any(map(_is_installability_error, p.get("notes", []))) for p in packages
)
summary = {
"total_downstream_packages": sum(int("git" in p) for p in packages),
"total_errors_count": sum(len(p.get("errors", [])) for p in packages),
"total_updates_count": sum(len(p.get("updates", [])) for p in packages),
"installability_errors_count": installability_errors_count,
"installability_notes_count": installability_notes_count,
}
data["summary"] = summary
def render_dashboard(env, data, destdir):
logging.debug(f"Generating {destdir}/index.html")
index = env.get_template("index.html.jinja2").render(**data)
with open(destdir / "index.html", "w") as index_file:
index_file.write(index)
def render_installability(env, data, destdir):
logging.debug(f"Generating {destdir}/installability.html")
causes = {}
for pkgname, p in data["packages"].items():
errors = p.get("errors", []) + p.get("notes", [])
for error in errors:
missing = error.get("details", [{}])[0].get("missing")
if not missing:
continue
pkg = missing["pkg"]
if "depchains" in missing:
binary = missing["depchains"][0]["depchain"][0]["package"]
else:
binary = pkg["package"]
cause = dict(error)
cause.update(binary=binary, dependency=pkg)
causes.setdefault(pkg["unsat-dependency"], []).append(cause)
installability = env.get_template("installability.html.jinja2").render(
causes=causes, summary=data["summary"]
)
with open(destdir / "installability.html", "w") as installability_file:
installability_file.write(installability)
def render(data, destdir):
destdir.mkdir(parents=True, exist_ok=True)
yaml.dump(
data, open(destdir / "data.yaml", "w"), Dumper=yaml.CSafeDumper, width=120
)
env = jinja2.Environment(loader=jinja2.FileSystemLoader("templates/"))
logging.debug(f"Generating {destdir}/index.html")
index = env.get_template("index.html.jinja2").render(**data)
with open(destdir / "index.html", "w") as index_file:
index_file.write(index)
env = jinja2.Environment(
loader=jinja2.FileSystemLoader("templates/"), autoescape=True
)
render_dashboard(env, data, destdir)
render_installability(env, data, destdir)
logging.debug(f"Copying CSS to {destdir}/css")
shutil.rmtree(
destdir / "css", ignore_errors=True
......
#!/usr/bin/env python3
import argparse
import io
import logging
import os
import pathlib
import re
import shutil
import subprocess
import tempfile
import coloredlogs
import debian.deb822
import yaml
from utils import fetch_packages_listing, thread_pool
DBGSYM_SUFFIX = "-dbgsym"
def _cores_available():
return len(os.sched_getaffinity(0))
def _drill(d, keys, default=None):
default = {} if default is None else default
for k in keys[:-1]:
d = d.setdefault(k, {})
d = d.setdefault(keys[-1], default)
return d
def _get_ignore_reason(report, rules):
binaryname = report["package"]
for rule in rules:
match = any(p.fullmatch(binaryname) for p in rule.get("patterns", []))
if match:
return rule["reason"]
return None
def _compile_patterns(rules):
for rule in rules:
patterns = rule.get("patterns", [])
for index, pattern in enumerate(patterns):
patterns[index] = re.compile(pattern)
def _intersect_architectures(channel, component_names, filter_architectures):
components = [channel["components"][c] for c in component_names]
architectures = set(components[0]["architectures"])
for component in components[1:]:
architectures.intersection_update(component["architectures"])
if filter_architectures:
architectures.intersection_update(filter_architectures)
return architectures
class InstallabilityChecker:
def __init__(self, filter_releases, filter_components, filter_architectures):
self.filter_releases = filter_releases
self.filter_components = filter_components
self.filter_architectures = filter_architectures
self.tmpdir = tempfile.TemporaryDirectory(prefix="installability-")
self.channels = None
self.packages = {}
self.binaries = {}
def _error(self, package_name, msg, **kwargs):
ignore_reason = kwargs.get("ignore_reason")
if ignore_reason:
dest = "notes"
logging.debug("note: %s: %s (%s)", package_name, msg, ignore_reason)
else:
dest = "errors"
logging.debug("error: %s: %s", package_name, msg)
assert package_name
assert msg
package = self.packages[package_name]
package.setdefault("name", package_name)
package.setdefault(dest, []).append(dict(msg=msg, **kwargs))
def _filtered_components(self):
for channel in self.channels.values():
release = channel["release"]
if self.filter_releases and release not in self.filter_releases:
continue
for fg, component in channel["components"].items():
yield channel, fg, component
def load_channels(self, channels_definitions_file):
with open(channels_definitions_file, "r") as f:
channels_definitions = yaml.load(f, Loader=yaml.CSafeLoader)
self.channels = channels_definitions["channels"]
def packagelistname(self, release, component, architecture):
return pathlib.Path(self.tmpdir.name, f"{release}_{component}_{architecture}")
def fetch_package_lists(self):
network_bound_threads_count = 10
source_items = []
binary_items = []
for channel, fg, component in self._filtered_components():
release = channel["release"]
architectures = [
arch
for arch in component["architectures"]
if not self.filter_architectures or arch in self.filter_architectures
]
source_items.append((release, fg))
for architecture in architectures:
binary_items.append((release, fg, architecture))
thread_pool(
network_bound_threads_count,
lambda item: self._fetch_source_list(*item),
source_items,
)
thread_pool(
network_bound_threads_count,
lambda item: self._fetch_binary_list(*item),
binary_items,
)
def _fetch_source_list(self, release, component):
logging.info(f"Fetching sources for {release}/{component}")
url = f"https://repositories.apertis.org/apertis/dists/{release}/{component}/source/Sources"
contents = io.TextIOWrapper(fetch_packages_listing(url), encoding="utf-8")
for p in debian.deb822.Packages.iter_paragraphs(contents, use_apt_pkg=False):
package = p["Package"]
binaries = {b.strip(): {} for b in p["Binary"].split(",")}
version = p["Version"]
logging.debug(f"Found source {release} {component} {package} {version}")
_drill(
self.packages,
(package, "apt", release, component),
{"version": version, "binaries": binaries},
)
for binary in binaries:
self.binaries[binary] = package
def _fetch_binary_list(self, release, component, arch):
logging.info(f"Fetching binaries for {release}/{component}/{arch}")
url = f"https://repositories.apertis.org/apertis/dists/{release}/{component}/binary-{arch}/Packages"
contents = fetch_packages_listing(url)
with open(self.packagelistname(release, component, arch), "w+b") as dest:
shutil.copyfileobj(contents, dest)
dest.seek(0)
for p in debian.deb822.Packages.iter_paragraphs(dest, use_apt_pkg=False):
binary = p["Package"]
source = p.get("Source", binary)
version = p["Version"]
logging.debug(
f"Found binary {release} {component} {arch} {binary} {version}"
)
_drill(
self.packages, (source, "apt", release, component, "binaries"), {}
).setdefault(binary, {})[arch] = version
if not binary.endswith(DBGSYM_SUFFIX) and binary not in self.binaries:
msg = f"Package {binary}/{arch} missing from source index on {release}/{component}"
self._error(source, msg)
self.binaries[binary] = source
def _check_scenario(self, release, components, architecture, ignore):
fg, *bgs = components
report = self._run_distcheck(release, architecture, fg, bgs)["report"] or []
for item in report:
reason = _get_ignore_reason(item, ignore)
binaryname = item.pop("package")
packagename = self.binaries.get(binaryname, binaryname)
_drill(
self.packages,
(packagename, "apt", release, fg, architecture),
{"version": item["version"]},
)
if item["status"] != "ok":
scenario = f"{release}/{'+'.join(components)}/{architecture}"
msg = f"Installability of package {binaryname} is {item['status']} on {scenario}"
extra = {
"details": item["reasons"],
"scenario": scenario,
}
if reason:
msg += f" [ignored: ({reason})]"
extra["ignore_reason"] = reason
self._error(packagename, msg, **extra)
logging.info(f"Processed {release}/{'+'.join(components)}/{architecture}")
def check_installability(self):
cpu_bound_threads_count = _cores_available()
items = []
for channel, fg, component in self._filtered_components():
if self.filter_components and fg not in self.filter_components:
continue
release = channel["release"]
for scenario in component["scenarios"]:
components = [fg, *scenario.get("background", [])]
ignore = scenario.get("ignore", [])
_compile_patterns(ignore)
architectures = _intersect_architectures(
channel, components, self.filter_architectures
)
logging.info(
f"Processing {release}/{'+'.join(components)} on {'/'.join(architectures)}"
)
for architecture in architectures:
items.append((release, components, architecture, ignore))
thread_pool(
cpu_bound_threads_count, lambda item: self._check_scenario(*item), items
)
def _run_distcheck(self, release, arch, fg, bgs):
cmd = [
"dose-distcheck",
"--verbose",
"--explain",
"--failures",
"--explain-condense",
"--deb-ignore-essential",
"--fg",
f"deb://{self.packagelistname(release, fg, arch)}",
]
for bg in bgs:
cmd.append("--bg")
cmd.append(f"deb://{self.packagelistname(release, bg, arch)}")
logging.debug(f"Running {' '.join(cmd)}")
cmd_output = subprocess.run(cmd, stdout=subprocess.PIPE, check=False)
logging.debug(
f"Completed dose-distcheck run for {release}/{'+'.join([fg, *bgs])}/{arch}"
)
return yaml.load(cmd_output.stdout, Loader=yaml.CSafeLoader)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Fetch data from the APT repositories and check dependency constraints"
)
parser.add_argument(
"--debug",
action="store_const",
dest="loglevel",
const=logging.DEBUG,
help="print debug information",
)
parser.add_argument(
"--quiet",
action="store_const",
dest="loglevel",
const=logging.WARNING,
help="do not print informational output",
)
parser.add_argument(
"--filter-release",
action="append",
help="limit the analysis to the selected releases",
)
parser.add_argument(
"--filter-component",
action="append",
help="limit the analysis to the selected components",
)
parser.add_argument(
"--filter-architecture",
action="append",
help="limit the analysis to the selected architectures",
)
parser.add_argument(
"--yaml",
required=True,
type=argparse.FileType("w"),
help="file to store results in YAML format",
)
args = parser.parse_args()
coloredlogs.install(level=args.loglevel or logging.INFO)
i = InstallabilityChecker(
args.filter_release, args.filter_component, args.filter_architecture
)
i.load_channels("data/channels.yaml")
i.fetch_package_lists()
i.check_installability()
data = {"packages": i.packages}
error_packages = set(p["name"] for p in data["packages"].values() if "errors" in p)
ignore_packages = set(p["name"] for p in data["packages"].values() if "notes" in p)
error_count = len(error_packages)
ignore_count = len(ignore_packages - error_packages)
print(
"📦" if not error_packages else "⚠️ ",
f"Retrieved {len(data['packages'])} packages with {error_count} errors and {ignore_count} ignored reports",
)
yaml.dump(data, args.yaml, width=120, Dumper=yaml.CSafeDumper)
......@@ -3,32 +3,18 @@
from __future__ import annotations
import argparse
import gzip
import io
import logging
import lzma
import debian.deb822
import debian.debian_support
import requests
import yaml
from classes import UpstreamPackage, UpstreamSource
from utils import fetch_packages_listing
def fetch_packages_listing(url):
decompressor = lzma
r = requests.get(url + ".xz", stream=True)
if r.status_code == requests.codes.not_found:
decompressor = gzip
r = requests.get(url + ".gz", stream=True)
if r.status_code == requests.codes.not_found:
decompressor = None
r = requests.get(url, stream=True)
r.raise_for_status()
contents = io.BytesIO(r.content)
if decompressor:
contents = decompressor.open(contents)
def fetch_packages(url):
fields = ("Package", "Version")
contents = fetch_packages_listing(url)
return list(
debian.deb822.Packages.iter_paragraphs(
contents, use_apt_pkg=False, fields=fields
......@@ -56,7 +42,7 @@ class UpstreamFetcher:
logging.info(
f"Fetching upstream packages for {source.destination} from {suite}"
)
for pkg in fetch_packages_listing(url):
for pkg in fetch_packages(url):
p = UpstreamPackage(
pkg["Package"],
pkg.get_version(),
......
import collections
import concurrent.futures
import gzip
import io
import logging
import lzma
import sys
import requests
def fetch_packages_listing(url):
decompressor = lzma
r = requests.get(url + ".xz", stream=True)
if r.status_code == requests.codes.not_found:
decompressor = gzip
r = requests.get(url + ".gz", stream=True)
if r.status_code == requests.codes.not_found:
decompressor = None
r = requests.get(url, stream=True)
r.raise_for_status()
contents = io.BytesIO(r.content)
if decompressor:
contents = decompressor.open(contents)
return contents
def item_id(item):
itemid = None
......
.defaults:
- &architectures [amd64, armhf, arm64]
- &ignore_target_images
- patterns:
- .*-dbg
- .*-dbgsym
reason: Debug symbols are not meant to be installed on pure target images
- patterns:
- .*-doc
reason: Documentation packages are not meant to be installed on pure target images
- patterns:
- .*-source-.*
- .*-source
reason: Source packages are not meant to be installed on pure target images
- patterns:
- .*-tests
- .*-tests-additions
reason: Test packages are not meant to be installed on pure target images
- patterns:
- .*-dev
- .*-headers(-.*)?
- libtool(-bin)?
- libc6-dev.*
- lib(32|64|n32|x32)stdc\+\+-.-dev-.*
- libstdc\+\+-.-dev-.*
reason: Development packages are not meant to be installed on pure target images
- patterns:
- linux-compiler-.*
- binutils-.*-linux-.*
- crossbuild-essential-.*
- clang.*
- lld-.*
- llvm-.*
- fixincludes
- cpp-.*
- g\+\+-.*
- gcc-.*
- gccbrig-.*
- gccgo-.*
- gdc-.*
- gfortran-.*
- gnat-.*
- gobjc-.*
- gobjc\+\+-.*
- libgccjit0
reason: Compiler executables are not meant to be installed on pure target images
- patterns:
- libapache2-mod-apparmor
reason: "Depends on apache2-api-20120211 provided by apache2-bin in development: drop?"
- patterns:
- python-.*
reason: Python2 is not available on targets
- patterns:
- caca-utils
reason: Depends on libimlib2 which is only in the development repositories
- patterns:
- dh-python
- python3-all
- libglib2.0-dev-bin
reason: Depends on python3-distutils which is only in the development repositories
- patterns:
- gstreamer1.0-qt5
- libpoppler-qt5-1
- qv4l2
reason: Depends on libqt5core5a which is only in the development repositories
- patterns:
- libopenjpip-server
reason: Depends on libwww-perl which is only in the development repositories
- patterns:
- llvm-.*-runtime
- llvm-.*-examples
reason: The LLVM interpreter and examples are not meant to be installed on pure target images
- patterns:
- gobject-introspection
reason: The GObject-Introspection development tools are not meant to be installed on pure target images
- patterns:
- dkms
reason: Kernel modules are not meant to be built on pure target images
- &components
target:
scenarios:
- ignore: *ignore_target_images
- background: [development, sdk]
architectures: *architectures
hmi:
scenarios:
- background: [target,]
ignore: *ignore_target_images
- background: [target, development, sdk]
architectures: *architectures
development:
scenarios:
- background: [target, hmi]
ignore:
- patterns:
- .*-doc
reason: Documentation packages are only meant to be installed on the full SDK
- background: [target, hmi, sdk]
architectures: *architectures
sdk:
scenarios:
- background: [target, development, hmi]
architectures: [amd64]
channels:
apertis/v2019:
source:
distribution: debian
release: buster
release: v2019
components: *components
apertis/v2020:
source:
distribution: debian
release: buster
release: v2020
components: *components
apertis/v2021pre: