From 73ce805c04d06951d536d6518c3da72b13989e78 Mon Sep 17 00:00:00 2001
From: Emanuele Aina <emanuele.aina@collabora.com>
Date: Sun, 29 Aug 2021 16:12:27 +0200
Subject: [PATCH 1/3] gitlab-ci: Fix $TRIGGER_UPDATES check
Repeat with me: `test -e` stands for "file exists", *not* for "string is
empty", as that's `test -z`.
Update the test in `trigger-updates` accordingly to properly print the
help text when the variable is not set.
Signed-off-by: Emanuele Aina <emanuele.aina@collabora.com>
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 5dcc2d8..c06fe85 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -230,7 +230,7 @@ trigger-updates:
wget --header "PRIVATE-TOKEN: $CI_JOB_TOKEN" "$CI_PROJECT_URL/-/jobs/$TRIGGER_FROM_JOB/artifacts/raw/packaging-updates.yaml"
fi
- |
- if [ -e "$TRIGGER_UPDATES" ]
+ if [ -z "$TRIGGER_UPDATES" ]
then
echo 'Set TRIGGER_UPDATES by manually triggering the pipeline to actually initiate the updates'
echo '* use "*" to match everything'
--
GitLab
From b89963f4a1f01ae4bd4935c3cdc146d1067d8a21 Mon Sep 17 00:00:00 2001
From: Emanuele Aina <emanuele.aina@collabora.com>
Date: Sun, 29 Aug 2021 11:53:27 +0200
Subject: [PATCH 2/3] gitlab-ci: Fix triggering updates when auth is required
When retrieving an artifacts from a past job with a job token, the
`JOB-TOKEN``header should be used rather than the `PRIVATE-HEADER` and
the API URL should be used rather than the enduser route.
Signed-off-by: Emanuele Aina <emanuele.aina@collabora.com>
---
.gitlab-ci.yml | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c06fe85..94cf699 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -227,7 +227,9 @@ trigger-updates:
if [ -n "$TRIGGER_FROM_JOB" ]
then
echo "Retrieving updatable packages from job $TRIGGER_FROM_JOB"
- wget --header "PRIVATE-TOKEN: $CI_JOB_TOKEN" "$CI_PROJECT_URL/-/jobs/$TRIGGER_FROM_JOB/artifacts/raw/packaging-updates.yaml"
+ JOB_ID=${TRIGGER_FROM_JOB##*/}
+ JOB_API_URL=$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$JOB_ID/
+ wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$JOB_API_URL/artifacts/packaging-updates.yaml"
fi
- |
if [ -z "$TRIGGER_UPDATES" ]
--
GitLab
From 3cba2e86829661139201c803e10a5a155e775d9b Mon Sep 17 00:00:00 2001
From: Emanuele Aina <emanuele.aina@collabora.com>
Date: Tue, 24 Aug 2021 09:31:30 +0200
Subject: [PATCH 3/3] Use the final data.yaml when triggering activities
The `public/data.yaml` generated by the dashboard contains all the
combined data to let triggers do their job, using it gives more
flexibility later on.
Signed-off-by: Emanuele Aina <emanuele.aina@collabora.com>
---
.gitlab-ci.yml | 9 ++++++---
bin/dashboard | 5 +++++
bin/trigger-updates | 10 +++++++---
templates/index.html.jinja2 | 6 +++---
4 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 94cf699..352e1a7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,8 +10,8 @@ stages:
- fetch
- check
- plan
- - trigger
- render
+ - trigger
.netrc-setup-snippet: &netrc-setup-snippet
- |
@@ -229,7 +229,9 @@ trigger-updates:
echo "Retrieving updatable packages from job $TRIGGER_FROM_JOB"
JOB_ID=${TRIGGER_FROM_JOB##*/}
JOB_API_URL=$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$JOB_ID/
- wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$JOB_API_URL/artifacts/packaging-updates.yaml"
+ wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$JOB_API_URL/artifacts/public/data.yaml"
+ else
+ cp public/data.yaml data.yaml
fi
- |
if [ -z "$TRIGGER_UPDATES" ]
@@ -243,7 +245,7 @@ trigger-updates:
- ./bin/trigger-updates
--gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
--gitlab-server-url "${CI_SERVER_URL}"
- --projects packaging-updates.yaml
+ --projects data.yaml
--filter "${TRIGGER_UPDATES}"
.render:
@@ -264,6 +266,7 @@ trigger-updates:
- ./bin/dashboard
--data packaging.yaml
--destdir public
+ --current-job-url "$CI_JOB_URL"
--current-pipeline-url "$CI_PIPELINE_URL"
--new-pipeline-url "$CI_PROJECT_URL/-/pipelines/new?ref=$CI_COMMIT_REF_NAME"
- ./bin/tsv
diff --git a/bin/dashboard b/bin/dashboard
index 74b7fb9..89afc61 100755
--- a/bin/dashboard
+++ b/bin/dashboard
@@ -58,6 +58,10 @@ if __name__ == "__main__":
const=logging.WARNING,
help="do not print informational output",
)
+ parser.add_argument(
+ "--current-job-url",
+ help="URL of the generating job",
+ )
parser.add_argument(
"--current-pipeline-url",
help="URL of the generating pipeline",
@@ -80,6 +84,7 @@ if __name__ == "__main__":
logging.debug(f"Loading YAML from {args.data.name}")
data = yaml.load(args.data, Loader=yaml.CSafeLoader)
data["meta"] = {
+ "current_job_url": args.current_job_url,
"current_pipeline_url": args.current_pipeline_url,
"new_pipeline_url": args.new_pipeline_url,
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
diff --git a/bin/trigger-updates b/bin/trigger-updates
index 2f61c16..7a5f8f9 100755
--- a/bin/trigger-updates
+++ b/bin/trigger-updates
@@ -23,19 +23,23 @@ def connect(gitlab_instance, gitlab_server_url, gitlab_api_token):
def trigger_updates(gl, data, filterglob):
all_packages = data["packages"]
- all_count = sum(len(package["updates"]) for package in all_packages.values())
+ all_count = sum(
+ len(package.get("updates", [])) for package in all_packages.values()
+ )
filtered_packages = [
p
for package_name, p in all_packages.items()
if fnmatch.fnmatch(package_name, filterglob)
]
- filtered_count = sum(len(package["updates"]) for package in filtered_packages)
+ filtered_count = sum(
+ len(package.get("updates", [])) for package in filtered_packages
+ )
logging.info(
f"Processing {filtered_count} updates matching the '{filterglob}' filter, {all_count} total"
)
for package_name, package in all_packages.items():
should_trigger = fnmatch.fnmatch(package_name, filterglob)
- for update in package["updates"]:
+ for update in package.get("updates", []):
path_with_namespace = package["git"]["path_with_namespace"]
ref = update["base"]["name"]
print(
diff --git a/templates/index.html.jinja2 b/templates/index.html.jinja2
index cd5261e..d1cfd3d 100644
--- a/templates/index.html.jinja2
+++ b/templates/index.html.jinja2
@@ -45,10 +45,10 @@
<div>
{% if meta.new_pipeline_url -%}
<a class="btn btn-primary {{'disabled' if not summary.total_updates_count }}" href="{{- meta.new_pipeline_url -}}&var[TRIGGER_UPDATES]=*
- {%- if meta.updates_job_id -%}
- &var[TRIGGER_FROM_JOB]={{- meta.updates_job_id -}}
+ {%- if meta.current_job_url -%}
+ &var[TRIGGER_FROM_JOB]={{- meta.current_job_url -}}
{%- endif -%}
- ">Trigger all updates</a>
+ ">Update all</a>
{%- endif %}
</div>
</div>
--
GitLab