Skip to content
Snippets Groups Projects
Commit c0c77cb6 authored by Emanuele Aina's avatar Emanuele Aina
Browse files

Use more structured reports 


Rather than using plaintext error messages, use readable codes and
structured metadata for errors and updates to make them easier
to process.

This will be particularly useful for filtering: for instance we
preserve the branch information rather than muddling it in the
error message.

Signed-off-by: Emanuele Aina's avatarEmanuele Aina <emanuele.aina@collabora.com>
parent d16a3adf
No related branches found
No related tags found
1 merge request!72Implement filtering by severity and release channel
Expand all Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
+ 2
− 0
View file @ c0c77cb6
......@@ -266,6 +266,7 @@ packaging-updates:
before_script:
- apt update && apt install -y --no-install-recommends
python3-debian
python3-gitlab
python3-yaml
script:
- ./bin/packaging-updates
......@@ -287,6 +288,7 @@ packaging-updates-upstream-linux:
ca-certificates
git
python3-debian
python3-gitlab
python3-yaml
script:
- ./bin/packaging-updates-upstream-linux
......
bin/classes.py
+ 34
− 0
View file @ c0c77cb6
import dataclasses
import enum
import typing
import debian.debian_support
......@@ -15,6 +16,39 @@ RENAMED = {
}
class Report(enum.Enum):
def _generate_next_value_(name, start, count, last_values):
return name.lower().replace("_", "-")
APT_PACKAGE_BINARIES_AMBIGUOUS = enum.auto()
APT_PACKAGE_BINARIES_MISSING = enum.auto()
APT_PACKAGE_MISSING = enum.auto()
APT_PACKAGE_MISSING_BUT_ON_OBS = enum.auto()
APT_PACKAGE_SOURCE_AMBIGUOUS = enum.auto()
APT_PACKAGE_SOURCE_MISSING = enum.auto()
APT_PACKAGE_SOURCE_VERSION_MISMATCH_BINARIES = enum.auto()
APT_PACKAGE_VERSION_MISMATCH_OBS = enum.auto()
GIT_BRANCH_COMPONENT_MISSING = enum.auto()
GIT_BRANCH_FOLDED_BUT_NOT_REMOVED = enum.auto()
GIT_BRANCH_HAS_AMBIGUOUS_TAGS = enum.auto()
GIT_BRANCH_LICENSING_REPORT_MISSING = enum.auto()
GIT_BRANCH_MISSING_BUT_ON_OBS = enum.auto()
GIT_BRANCH_NOT_POINTING_TO_TAGGED_COMMIT = enum.auto()
GIT_BRANCH_PIPELINE_FAILED = enum.auto()
GIT_CHANNEL_LAGGING = enum.auto()
GIT_PROJECT_MISSING = enum.auto()
GIT_UPSTREAM_BRANCH_DROPPED = enum.auto()
GIT_UPSTREAM_BRANCH_NOT_MERGED = enum.auto()
OBS_PACKAGE_AMBIGUOUS = enum.auto()
OBS_PACKAGE_BUILD_FAILED = enum.auto()
OBS_PACKAGE_MISSING_BUT_IN_GIT = enum.auto()
OBS_PACKAGE_MISSING_BUT_ON_APT = enum.auto()
OBS_PACKAGE_MISSING_BUT_PUBLISHED = enum.auto()
OBS_PACKAGE_VERSION_MISMATCH = enum.auto()
UPDATE_AVAILABLE = enum.auto()
UPDATE_AVAILABLE_MAINLINE = enum.auto()
@dataclasses.dataclass
class UpstreamPackage:
name: str
......
bin/dashboard
+ 12
− 2
View file @ c0c77cb6
......@@ -11,12 +11,22 @@ import jinja2
import yaml
def count_reports(package, func):
reports = package.get("reports", [])
return sum(1 for r in reports if func(r))
def preprocess_packaging_data(data):
packages = data["packages"].values()
summary = {
"total_downstream_packages": sum(int("git" in p) for p in packages),
"total_errors_count": sum(len(p.get("errors", [])) for p in packages),
"total_updates_count": sum(len(p.get("updates", [])) for p in packages),
"total_errors_count": sum(
count_reports(p, lambda r: r["severity"] == "error") for p in packages
),
"total_updates_count": sum(
count_reports(p, lambda r: r["kind"].startswith("update-available"))
for p in packages
),
}
data["summary"] = summary
......
bin/packaging-check-invariants
+ 165
− 72
View file @ c0c77cb6
This diff is collapsed.
bin/packaging-updates
+ 5
− 2
View file @ c0c77cb6
......@@ -8,6 +8,7 @@ import types
import debian.debian_support
import yaml
from classes import Report
def base_for_source(data, sourcename):
......@@ -36,14 +37,16 @@ def compute_updates(data):
msg += f" can be updated to {upstream['version']}"
logging.info(msg)
update = {
"branch": {"name": branch.name, "version": branch.version},
"kind": Report.UPDATE_AVAILABLE.value,
"severity": "info",
"branch": branch.name,
"upstream": upstream,
}
if base:
update["base"] = {"name": base.name, "version": base.version}
p = ret["packages"].setdefault(package.name, {})
p["git"] = {"path_with_namespace": package.git["path_with_namespace"]}
p.setdefault("updates", []).append(update)
p.setdefault("reports", []).append(update)
for package in data["packages"].values():
if "git" not in package:
......
bin/packaging-updates-upstream-linux
+ 32
− 19
View file @ c0c77cb6
......@@ -10,6 +10,7 @@ from collections import defaultdict
import debian.debian_support
import yaml
from classes import Report
def base_latest_version(entries, package):
......@@ -48,13 +49,23 @@ def compute_linux_updates(data):
logging.debug("No entry for linux found in the packaging data, skipping")
return ret
def error(package, err):
name = package["name"]
def report(severity, packagename, kind, **kwargs):
logfunc = getattr(logging, severity)
logfunc(
"%s: %s: %s",
packagename,
kind.value,
", ".join(f"{k}={v}" for k, v in kwargs.items()),
)
packages = ret["packages"]
p = packages.setdefault(name, {})
logging.error(f"{name}: {err['branch']}: {err['msg']}")
errors = p.setdefault("errors", [])
errors.append(err)
reports = packages.setdefault(packagename, {}).setdefault("reports", [])
reports.append(dict(kind=kind.value, severity=severity, **kwargs))
def update(packagename, kind, **kwargs):
report("info", packagename, kind, **kwargs)
def error(packagename, kind, **kwargs):
report("error", packagename, kind, **kwargs)
branch_versions = {}
url = "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git"
......@@ -87,11 +98,12 @@ def compute_linux_updates(data):
)
downstream_version = debian.debian_support.Version(version)
if downstream_version < debian_version:
err = {
"branch": debian_base,
"msg": f"Branch {debian_base}={debian_version} has not been merged into {branch}={version}",
}
error(package, err)
error(
package["name"],
Report.GIT_UPSTREAM_BRANCH_NOT_MERGED,
branch=debian_base,
downstreams=[branch],
)
continue
# Fetch the versions available upstream for the downstream tag (for e.g. 5.4) using
......@@ -128,15 +140,16 @@ def compute_linux_updates(data):
logging.debug(f"{branch}: Downstream has {version}, git has {latest_tag}")
downstream_version = debian.debian_support.Version(version)
if downstream_version < latest_tag:
err = {
"branch": branch,
"msg": f"Branch {branch}={version} lags behind upstream {latest_tag}",
"git": {
"url": url,
"tag": f"v{latest_tag}",
},
mainline = {
"url": f"{url}/tag/?h=v{latest_tag}",
"tag": f"v{latest_tag}",
}
error(package, err)
update(
package["name"],
Report.UPDATE_AVAILABLE_MAINLINE,
branch=branch,
mainline=mainline,
)
return ret
......
templates/index.html.jinja2
+ 179
− 37
View file @ c0c77cb6
{% extends "base.html.jinja2" %}
{%- macro badge_class(severity) -%}
{%- if severity == "error" -%}
danger
{%- else -%}
{{ severity }}
{%- endif -%}
{%- endmacro -%}
{%- macro branch_link(package, branchname, show_version=False, classes="") -%}
{%- if "git" in package -%}
<a class="{{classes}}" href="{{ package.git.web_url }}/-/commits/{{ branchname }}">{{ branchname }}</a>
{%- if show_version and branchname in package.git.branches and package.git.branches[branchname].version -%}
/<code>{{ package.git.branches[branchname].version }}</code>
{%- endif -%}
{%- else -%}
<span class="{{classes}}">{{ branchname }}</span>
{%- endif -%}
{%- endmacro -%}
{%- macro branch(package, branchname) -%}
<span class="branch">{{ branch_link(package, branchname, show_version=True) }}</span>
{%- endmacro -%}
{%- macro obsproject(package, project, version=None) -%}
<span class="obsproject">
<a href="{{ obs[project].web_url }}">{{ project }}</a>/
{%- if "obs" in package and project in package.obs -%}
<a href="{{ package.obs[project].web_url }}">
{{- package.name -}}
{%- if version -%}
/<code>{{ version }}</code>
{%- endif -%}
</a>
{%- else -%}
{{ package.name }}
{%- endif -%}
</span>
{%- endmacro -%}
{%- macro obsprojects(package, projects) -%}
<span class="obsprojects">
{%- for project in projects -%}
{{ obsproject(package, project) }}
{%- endfor -%}
</span>
{%- endmacro -%}
{%- macro aptsources(package, sources) -%}
<span class="aptsources">
{%- for source in sources -%}
{{ source.component }}/<code>{{ source.version }}</code>
{%- if not loop.last %}, {% endif -%}
{%- endfor -%}
</span>
{%- endmacro -%}
{%- macro pipeline(package, pipeline_web_url) -%}
{%- if pipeline_web_url -%}
<span class="pipeline">
<a href="{{ pipeline_web_url }}">pipeline 🚀</a>
</span>
{%- endif -%}
{%- endmacro -%}
{% block title %} Packages {% endblock %}
{% block summary %}
......@@ -36,9 +100,11 @@
{% block content %}
<div class="list-group">
{% for package in packages.values() %}
{% if package.errors or package.updates %}
<div id="pkg-{{package.name}}" class="list-group-item list-group-item-action flex-column align-items-start">
{% for package in packages.values() if package.reports or package.updates %}
<div
id="pkg-{{package.name}}"
data-package="{{package.name}}"
class="package list-group-item list-group-item-action flex-column align-items-start">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">
<strong>
......@@ -52,43 +118,119 @@
</h5>
</div>
{% for error in package.errors %}
<p class="mb-1">
<big><span class="badge badge-danger">error</span></big>
{% if error.branch %}
Branch <a href="{{ package.git.web_url }}/-/commits/{{ error.branch }}">{{ error.branch }}</a>
{% if package.git.branches[error.branch].version %}
(<code>{{ package.git.branches[error.branch].version }}</code>)
{%- endif %}:
{% endif %}
{% if error.projects %}
Projects
{% for project in error.projects %}
<a href="{{ obs[project].web_url }}">{{ project }}</a>/<a href="{{ package.obs[project].web_url }}">{{ package.name }}</a>
{%- if not loop.last %},{% endif %}
{%- endfor %}:
{% endif %}
{{ error.msg }}
{%- if error.pipeline_web_url -%}
: <a href="{{ error.pipeline_web_url }}">pipeline 🚀</a>
{% endif %}
</p>
{% endfor %}
{% for report in package.reports %}
{%- set report_channels = [] -%}
{%- if report.kind == "git-upstream-branch-not-merged" -%}
{%- for downstream in report.downstreams -%}
{{- report_channels.append(channels[downstream].base) or "" -}}
{%- endfor -%}
{%- elif report.kind == "update-available" -%}
{%- for channel in channels.values() if channel.source.distribution+"/"+channel.source.release == sources[report.upstream.source].base -%}
{{- report_channels.append(channel.base) or "" -}}
{%- endfor -%}
{%- elif report.branch in sources -%}
{%- for channel in channels.values() if channel.source.distribution+"/"+channel.source.release == sources[report.branch].base -%}
{{- report_channels.append(channel.base) or "" -}}
{%- endfor -%}
{%- elif report.branch in channels -%}
{{- report_channels.append(channels[report.branch].base) or "" -}}
{%- endif -%}
<p
data-report="{{ report.kind }}"
data-severity="{{ report.severity }}"
data-channels="{{- report_channels|sort|unique|join(" ") -}}"
class="mb-1 report">
<big>
<span class="report-severity badge badge-{{ badge_class(report.severity) }} border border-{{ badge_class(report.severity) }}">
{% if report.kind.startswith("update-") -%}
update
{%- else -%}
{{ report.severity }}
{%- endif %}
</span>
</big>
{% for channel in (report_channels + [report.branch])|select|sort|unique -%}
<big>
{{ branch_link(package, channel, classes="report-branch badge badge-light border border-secondary text-muted") }}
</big>
{%- endfor %}
{% for obsproject in (report.get("obsprojects") or [report.get("obsproject")]) if obsproject -%}
<big>
<a class="report-obsproject badge badge-light border border-secondary text-muted" href="{{obs[obsproject].web_url}}">{{ obsproject }}</a>
</big>
{%- endfor %}
{% for u in package.updates %}
<p class="mb-1"><big><span class="badge badge-info">update</span></big>
Branch {{u.branch.name}} can be updated from
<a href="{{ package.git.web_url }}/pipelines/new?ref={{ (u.base or u.branch).name|urlencode }}">
<code>{{u.branch.version}}</code>
to <code>{{u.upstream.version}}</code></a> from {{u.upstream.source}} {{u.upstream.component}}
</p>
{% if report.kind == "apt-package-missing" -%}
Not published on APT
{%- elif report.kind == "apt-package-missing-but-on-obs" -%}
Not published on APT, found on OBS: {{ obsprojects(package, report.obsprojects) }}
{%- elif report.kind == "apt-package-version-mismatch-obs" -%}
Mismatch between APT {{ report.component }}/<code>{{ report.aptversion}}</code> and OBS {{ obsproject(package, report.obsproject, report.obsversion ) }} ({{ branch(package, report.branch) }})
{%- elif report.kind == "apt-package-source-version-mismatch-binaries" -%}
Mismatch between APT source {{ aptsources(package, [report.source]) }} and binary
{{report.binary.component}}/{{report.binary.name}}/<code>{{report.binary.version}}</code>
on {{ report.binary.architectures|join(", ") }} ({{ branch(package, report.branch) }})
{%- elif report.kind == "apt-package-source-missing" -%}
Sources not published on APT ({{ branch(package, report.branch) }})
{%- elif report.kind == "apt-package-source-ambiguous" -%}
Ambiguous sources on APT: {{ aptsources(package, report.sources) }} ({{ branch(package, report.branch) }})
{%- elif report.kind == "apt-package-binaries-missing" -%}
Binaries not published on APT ({{ branch(package, report.branch) }})
{%- elif report.kind == "apt-package-binaries-ambiguous" -%}
Ambiguous binaries on APT: {{report.components|join(", ")}} ({{ branch(package, report.branch) }})
{%- elif report.kind == "git-channel-lagging" -%}
Branch {{ branch(package, report.branch) }} lags behind {{ branch(package, report.latest_branch) }}
{%- elif report.kind == "git-branch-folded-but-not-removed" -%}
Branch {{ branch(package, report.branch) }} folded in {{ branch(package, report.folded_in) }} but not removed
{%- elif report.kind == "git-branch-has-ambiguous-tags" -%}
Branch {{ branch(package, report.branch) }} has ambiguous tags: {{report.tags|join(", ")}}
{%- elif report.kind == "git-branch-missing-but-on-obs" -%}
Branch {{ report.branch }} missing but found on OBS: {{ obsproject(package, report.obsproject) }}
{%- elif report.kind == "git-branch-not-pointing-to-tagged-commit" -%}
Branch {{branch(package, report.branch)}} not pointing to a tagged commit
{%- elif report.kind == "git-project-missing" -%}
GitLab project missing but found on OBS: {{ obsprojects(package, report.obsprojects) }}
{%- elif report.kind == "git-upstream-branch-not-merged" -%}
Upstream {{branch(package, report.branch)}} not merged in
{% for downstream in report.downstreams -%}
{{branch(package, downstream)}}
{%- if not loop.last %}, {% endif -%}
{%- endfor -%}
{%- elif report.kind == "git-upstream-branch-dropped" -%}
Upstream branch {{branch(package, report.branch)}} dropped
{%- elif report.kind == "git-branch-pipeline-failed" -%}
Pipeline failed on {{ branch(package, report.branch) }}: {{ pipeline(package, report.pipeline_web_url) }}
{%- elif report.kind == "git-branch-licensing-report-missing" -%}
Missing licensing report on {{branch(package, report.branch)}}
{%- elif report.kind == "git-branch-component-missing" -%}
Missing component on {{branch(package, report.branch)}}
{%- elif report.kind == "obs-package-ambiguous" -%}
Ambiguous package on OBS: {{ obsprojects(package, report.projects) }}
{%- elif report.kind == "obs-package-missing-but-on-apt" -%}
Missing on OBS {{ obsproject(package, report.obsproject) }}, found on APT in {{ aptsources(package, [report.source]) }} ({{ branch(package, report.branch) }})
{%- elif report.kind == "obs-package-missing-but-in-git" -%}
Missing on OBS {{ obsproject(package, report.obsproject) }}, found on Git {{ branch(package, report.branch) }}
{%- elif report.kind == "obs-package-version-mismatch" -%}
Mismatch between OBS {{ obsproject(package, report.obsproject, report.obsversion ) }} and Git {{ branch(package, report.branch) }}
{%- elif report.kind == "obs-package-build-failed" -%}
Failed to build on OBS {{ obsproject(package, report.obsproject) }} {{report.repository}}/{{report.architecture}}:
&lt;<code class="build-result-code">{{report.result.code}}</code>&gt;
{%- if "details" in report.result -%}
<span class="build-result-details">{{ report.result.details }}</span>
{%- endif -%}
{%- elif report.kind == "update-available" -%}
Branch {{branch(package, report.branch)}} can be updated
<a href="{{ package.git.web_url }}/pipelines/new?ref={{ (report.base or report.branch).name|urlencode }}">
to <code>{{report.upstream.version}}</code> from {{report.upstream.source}} {{report.upstream.component}} 🚀
</a>
{%- elif report.kind == "update-available-mainline" -%}
Branch {{branch(package, report.branch)}} lags behind mainline <a href="{{report.mainline.url}}">{{report.mainline.tag}}</a>
{%- else -%}
Unknown report: {{ report }}
{%- endif %}
</p>
{% endfor %}
</div>
{% endif %}
{% endfor %}
</div>
{% endblock %}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
Apertis Website Terms of Use Privacy Policy