From cc12f6bdbbc63a3c5f3fcef40a87e5c57bedfcec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dylan=20A=C3=AFssi?= <dylan.aissi@collabora.com>
Date: Thu, 18 Jul 2024 14:36:30 +0200
Subject: [PATCH 1/2] Split pipeline in two parts: one for building dashboard,
the other one for triggering jobs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This allows to quickly run trigger-* jobs without waiting for
the dashboard to be built (which takes a while).
Signed-off-by: Dylan Aïssi <dylan.aissi@collabora.com>
---
.gitlab-ci.yml | 545 +-----------------------------------
.gitlab-ci/dashboard.yml | 470 +++++++++++++++++++++++++++++++
.gitlab-ci/trigger-jobs.yml | 70 +++++
3 files changed, 543 insertions(+), 542 deletions(-)
create mode 100644 .gitlab-ci/dashboard.yml
create mode 100644 .gitlab-ci/trigger-jobs.yml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d92e1ea..61f89f2 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -79,6 +79,9 @@ variables:
- "yes"
- "no"
+include:
+ - local: .gitlab-ci/dashboard.yml
+ - local: .gitlab-ci/trigger-jobs.yml
stages:
- lint
@@ -88,14 +91,6 @@ stages:
- render
- trigger
-.netrc-setup-snippet: &netrc-setup-snippet
- - |
- if [ -f "$NETRC" ]
- then
- echo "Configuring credentials from the NETRC CI variable"
- cp "$NETRC" ~/.netrc
- fi
-
lint:
stage: lint
tags:
@@ -132,537 +127,3 @@ localtest:
- if: $CI_PIPELINE_SOURCE == "schedule"
allow_failure: true
- if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-packaging-data-fetch-downstream:
- resource_group: gitlab
- stage: fetch
- timeout: 1h 30m
- tags:
- - lightweight
- before_script:
- - apt update && apt install -y --no-install-recommends
- ca-certificates
- python3-debian
- python3-gitlab
- python3-tenacity
- python3-yaml
- wget
- script:
- - CACHE_ARGS=""
- - ARTIFACT_URL=${ARTIFACT_URL:-$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/$CI_DEFAULT_BRANCH/raw/packaging-cache.json?job=pages}
- - |
- if [ "$ARTIFACT_URL" != none ] && [ "$DISABLE_CACHE" == "no" ]
- then
- wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$ARTIFACT_URL" -O cache.json || true
- fi
- - |
- if [ -s cache.json ] && [ "$DISABLE_CACHE" == "no" ]
- then
- echo Load cacheable data from cache.json
- CACHE_ARGS="--cache cache.json"
- fi
- - ./bin/packaging-data-fetch-downstream
- --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
- --gitlab-server-url "${CI_SERVER_URL}"
- --filter "${PROJECTS_NAMESPACE}/${FILTER_PACKAGES}"
- --json packaging-data-downstream.json
- ${CACHE_ARGS}
- ${DEBUG:+--debug}
- ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
- artifacts:
- when: always
- paths:
- - packaging-data-downstream.json
- - ${LOG_TO_FILE}
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-packaging-data-fetch-sources-upstream:
- stage: fetch
- tags:
- - lightweight
- before_script:
- - apt update && apt install -y --no-install-recommends
- ca-certificates
- python3-debian
- python3-gitlab
- python3-requests
- python3-yaml
- wget
- script:
- - CACHE_ARGS=""
- - ARTIFACT_URL=${ARTIFACT_URL:-$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/$CI_DEFAULT_BRANCH/raw/packaging-cache.json?job=pages}
- - |
- if [ "$ARTIFACT_URL" != none ] && [ "$DISABLE_CACHE" == "no" ] && [ "$FILTER_ON_CACHE" == "yes" ]
- then
- wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$ARTIFACT_URL" -O cache.json || true
- fi
- - |
- if [ -s cache.json ] && [ "$DISABLE_CACHE" == "no" ] && [ "$FILTER_ON_CACHE" == "yes" ]
- then
- echo Load cacheable data from cache.json
- CACHE_ARGS="--cache cache.json"
- fi
- - ./bin/packaging-data-fetch-sources
- --sources-definitions data/sources.yaml
- --filter-packages "$FILTER_PACKAGES"
- --json packaging-data-sources-upstream.json
- ${CACHE_ARGS}
- ${DEBUG:+--debug}
- ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
- artifacts:
- when: always
- paths:
- - packaging-data-sources-upstream.json
- - ${LOG_TO_FILE}
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-packaging-data-fetch-sources-published:
- stage: fetch
- tags:
- - lightweight
- before_script:
- - *netrc-setup-snippet
- - apt update && apt install -y --no-install-recommends
- ca-certificates
- python3-debian
- python3-gitlab
- python3-requests
- python3-yaml
- script:
- - ./bin/packaging-data-fetch-sources
- --sources-definitions data/channels.yaml
- --sources-key channels
- --output-key published
- --filter-packages "$FILTER_PACKAGES"
- --no-output-sources-definitions
- --json packaging-data-sources-published.json
- ${DEBUG:+--debug}
- ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
- artifacts:
- when: always
- paths:
- - packaging-data-sources-published.json
- - ${LOG_TO_FILE}
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-packaging-data-fetch-binaries-published:
- stage: fetch
- tags:
- - lightweight
- before_script:
- - *netrc-setup-snippet
- - apt update && apt install -y --no-install-recommends
- ca-certificates
- python3-debian
- python3-gitlab
- python3-requests
- python3-yaml
- script:
- - ./bin/packaging-data-fetch-binaries
- --sources-definitions data/channels.yaml
- --sources-key channels
- --output-key published
- --filter-packages "$FILTER_PACKAGES"
- --json packaging-data-binaries-published.json
- ${DEBUG:+--debug}
- ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
- artifacts:
- when: always
- paths:
- - packaging-data-binaries-published.json
- - ${LOG_TO_FILE}
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-packaging-data-fetch-obs:
- resource_group: obs
- stage: fetch
- timeout: 3h
- tags:
- - lightweight
- before_script:
- - apt update && apt install -y --no-install-recommends
- ca-certificates
- python3-debian
- python3-gitlab
- python3-m2crypto
- python3-tenacity
- python3-yaml
- osc
- script:
- - ./bin/packaging-data-fetch-obs
- --oscrc "$DASHBOARD_OSCRC"
- --filter-packages "$FILTER_PACKAGES"
- --json packaging-data-obs.json
- ${DEBUG:+--debug}
- ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
- artifacts:
- when: always
- paths:
- - packaging-data-obs.json
- - ${LOG_TO_FILE}
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-storage-usage:
- stage: fetch
- tags:
- - lightweight
- cache:
- - key:
- files:
- - storage_stats/Cargo.lock
- paths:
- - storage_stats/cargo/bin
- - storage_stats/cargo/registry/index
- - storage_stats/cargo/registry/cache
- - storage_stats/cargo/git/db
- - storage_stats/target
- before_script:
- - apt update && apt install -y --no-install-recommends
- ca-certificates
- cargo
- libssl-dev
- pkg-config
- script:
- - cd storage_stats
- - export CARGO_HOME=$PWD/cargo
- - test -n "$DEBUG" && RUST_LOG=debug
- - cargo run --release --
- --sources-definitions ../data/channels.yaml
- --sources-key channels
- --yaml ../storage.yaml
- artifacts:
- paths:
- - storage.yaml
- timeout: 3h
- rules:
- - if: $SKIP_STORAGE_USAGE == "yes"
- when: never
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-packaging-check-settings:
- resource_group: gitlab
- stage: check
- timeout: 15m
- tags:
- - lightweight
- before_script:
- - echo 'deb http://deb.debian.org/debian bookworm-backports main' > /etc/apt/sources.list.d/bookworm-backports.list
- - apt update && apt install -y --no-install-recommends
- ca-certificates
- git
- python3-debian
- gitlab-rulez/bookworm-backports # required for `--output json`
- script:
- - GITLAB_RULES_GIT=$(echo $GITLAB_RULES_GIT | sed "s,/$CI_SERVER_HOST,/gitlab-ci-token:$CI_JOB_TOKEN@$CI_SERVER_HOST,")
- - echo Retrieving rules $GITLAB_RULES_FILE from $GITLAB_RULES_GIT
- - git clone --depth=1 "$GITLAB_RULES_GIT" rules
- - ./bin/packaging-check-settings
- --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
- --gitlab-server-url "${CI_SERVER_URL}"
- --filter "${PROJECTS_NAMESPACE}/${FILTER_PACKAGES}"
- --rules "rules/$GITLAB_RULES_FILE"
- --json packaging-check-settings.json
- ${DEBUG:+--debug}
- artifacts:
- when: always
- paths:
- - packaging-check-settings.json
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-packaging-check-delta:
- resource_group: gitlab
- stage: check
- timeout: 4h
- tags:
- - lightweight
- before_script:
- - apt update && apt install -y --no-install-recommends
- git
- python3-debian
- python3-gitlab
- python3-yaml
- wget
- script:
- - CACHE_DELTA_ARGS=""
- - ARTIFACT_URL=${ARTIFACT_URL:-$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/$CI_DEFAULT_BRANCH/raw/packaging-cache.json?job=pages}
- - |
- if [ "$ARTIFACT_URL" != none ] && [ "$DISABLE_CACHE" == "no" ]
- then
- wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$ARTIFACT_URL" -O cache.json || true
- fi
- - |
- if [ -s cache.json ] && [ "$DISABLE_CACHE" == "no" ]
- then
- echo Load cacheable data from cache.json
- CACHE_DELTA_ARGS="--cache cache.json"
- fi
- - ./bin/json-merge
- --input packaging-data-downstream.json
- --input packaging-data-sources-upstream.json
- --input packaging-data-sources-published.json
- --input packaging-data-binaries-published.json
- --input packaging-data-obs.json
- --output packaging-data.json
- - ./bin/packaging-check-delta
- --projects packaging-data.json
- --whitelists data/whitelists.yaml
- --json packaging-check-delta.json
- ${CACHE_DELTA_ARGS}
- ${DEBUG:+--debug}
- ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
- artifacts:
- when: always
- paths:
- - packaging-check-delta.json
- - ${LOG_TO_FILE}
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-packaging-check-invariants:
- stage: check
- tags:
- - lightweight
- before_script:
- - apt update && apt install -y --no-install-recommends
- python3-debian
- python3-gitlab
- python3-yaml
- script:
- - ./bin/json-merge
- --input packaging-data-downstream.json
- --input packaging-data-sources-upstream.json
- --input packaging-data-sources-published.json
- --input packaging-data-binaries-published.json
- --input packaging-data-obs.json
- --output packaging-data.json
- - ./bin/packaging-check-invariants
- --projects packaging-data.json
- --whitelists data/whitelists.yaml
- --json packaging-checks.json
- ${DEBUG:+--debug}
- ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
- artifacts:
- when: always
- paths:
- - packaging-data.json
- - packaging-checks.json
- - ${LOG_TO_FILE}
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-packaging-updates:
- stage: plan
- tags:
- - lightweight
- before_script:
- - apt update && apt install -y --no-install-recommends
- devscripts
- git
- python3-debian
- python3-gitlab
- python3-yaml
- script:
- - ./bin/packaging-updates
- --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
- --projects packaging-data.json
- --whitelists data/whitelists.yaml
- --json packaging-updates.json
- ${DEBUG:+--debug}
- ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
- artifacts:
- when: always
- paths:
- - packaging-updates.json
- - ${LOG_TO_FILE}
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-packaging-updates-upstream-linux:
- stage: plan
- tags:
- - lightweight
- before_script:
- - apt update && apt install -y --no-install-recommends
- ca-certificates
- git
- python3-debian
- python3-gitlab
- python3-yaml
- script:
- - ./bin/packaging-updates-upstream-linux
- --projects packaging-data.json
- --json packaging-updates-upstream-linux.json
- ${DEBUG:+--debug}
- ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
- artifacts:
- when: always
- paths:
- - packaging-updates-upstream-linux.json
- - ${LOG_TO_FILE}
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-.render:
- stage: render
- tags:
- - lightweight
- before_script:
- - apt update && apt install -y --no-install-recommends
- python3-jinja2
- python3-yaml
- tree
- script:
- - STORAGE_DATA_ARGS=""
- - |
- if [ -e storage.yaml ]
- then
- echo Load storage data from storage.yaml
- STORAGE_DATA_ARGS="--storage-data storage.yaml"
- fi
- - ./bin/json-merge
- --input packaging-data.json
- --input packaging-checks.json
- --input packaging-check-delta.json
- --input packaging-check-settings.json
- --input packaging-updates.json
- --input packaging-updates-upstream-linux.json
- --output packaging.json
- --output-cache packaging-cache.json
- - ./bin/dashboard
- --packaging-data packaging.json
- ${STORAGE_DATA_ARGS}
- --destdir public
- --current-job-url "$CI_JOB_URL"
- --current-pipeline-url "$CI_PIPELINE_URL"
- --new-pipeline-url "$CI_PROJECT_URL/-/pipelines/new?ref=$CI_COMMIT_REF_NAME"
- --os-website-url "${OS_WEBSITE}"
- - ./bin/tsv
- --data packaging.json
- --destdir tsv/
- - cd tsv && tree -H '.' -L 1 --noreport --charset utf-8 > index.html && cd ..
- - mv ./tsv public/
- artifacts:
- when: always
- expose_as: "Rendered dashboard"
- paths:
- - public/
- - packaging-cache.json
-
-test:
- extends:
- - .render
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-pages:
- extends:
- - .render
- rules:
- - if: $TRIGGER_FROM_JOB
- when: never
- - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_BRANCH
- when: never
- - if: $CI_PIPELINE_SOURCE != "merge_request_event"
-
-trigger-updates:
- interruptible: false
- retry: 0
- stage: trigger
- tags:
- - lightweight
- before_script:
- - apt update && apt install -y --no-install-recommends
- python3-gitlab
- python3-yaml
- wget
- script:
- - |
- if [ -n "$TRIGGER_FROM_JOB" ]
- then
- echo "Retrieving updatable packages from job $TRIGGER_FROM_JOB"
- JOB_ID=${TRIGGER_FROM_JOB##*/}
- JOB_API_URL=$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$JOB_ID/
- wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$JOB_API_URL/artifacts/public/packaging.json"
- else
- cp public/packaging.json .
- fi
- - |
- if [ -z "$TRIGGER_UPDATES" ]
- then
- echo 'Set TRIGGER_UPDATES by manually triggering the pipeline to actually initiate the updates'
- echo '* use "*" to match everything'
- echo '* use "dash" to only process the dash package'
- echo 'If TRIGGER_UPDATES is left empty, do a dry run (this is the default).'
- echo "For instance: $CI_PROJECT_URL/-/pipelines/new?var[TRIGGER_UPDATES]=*&ref=$CI_COMMIT_REF_NAME"
- fi
- - ./bin/trigger-updates
- --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
- --gitlab-server-url "${CI_SERVER_URL}"
- --projects packaging.json
- --whitelists data/whitelists.yaml
- --filter "${TRIGGER_UPDATES}"
- ${TRIGGER_SECURITY_UPDATES:+--only-security}
- ${DEBUG:+--debug}
- ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
- artifacts:
- when: always
- paths:
- - ${LOG_TO_FILE}
-
-trigger-gitlab-rulez:
- interruptible: false
- retry: 0
- stage: trigger
- tags:
- - lightweight
- rules:
- - if: $TRIGGER_GITLAB_RULEZ == "apply"
- before_script:
- - apt update && apt install -y --no-install-recommends
- gitlab-rulez
- wget
- script:
- - GITLAB_RULES_GIT=$(echo $GITLAB_RULES_GIT | sed "s,/$CI_SERVER_HOST,/gitlab-ci-token:$CI_JOB_TOKEN@$CI_SERVER_HOST,")
- - echo Retrieving rules $GITLAB_RULES_FILE from $GITLAB_RULES_GIT
- - git clone --depth=1 "$GITLAB_RULES_GIT" rules
- - gitlab-rulez
- --gitlab-server-url "${CI_SERVER_URL}"
- --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
- "${TRIGGER_GITLAB_RULEZ}" "rules/$GITLAB_RULES_FILE"
- --filter "${PROJECTS_NAMESPACE}/${FILTER_PACKAGES}" | tee log_gitlab_rulez.txt
- artifacts:
- when: always
- paths:
- - log_gitlab_rulez.txt
diff --git a/.gitlab-ci/dashboard.yml b/.gitlab-ci/dashboard.yml
new file mode 100644
index 0000000..fa61917
--- /dev/null
+++ b/.gitlab-ci/dashboard.yml
@@ -0,0 +1,470 @@
+.netrc-setup-snippet: &netrc-setup-snippet
+ - |
+ if [ -f "$NETRC" ]
+ then
+ echo "Configuring credentials from the NETRC CI variable"
+ cp "$NETRC" ~/.netrc
+ fi
+
+packaging-data-fetch-downstream:
+ resource_group: gitlab
+ stage: fetch
+ timeout: 1h 30m
+ tags:
+ - lightweight
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ ca-certificates
+ python3-debian
+ python3-gitlab
+ python3-tenacity
+ python3-yaml
+ wget
+ script:
+ - CACHE_ARGS=""
+ - ARTIFACT_URL=${ARTIFACT_URL:-$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/$CI_DEFAULT_BRANCH/raw/packaging-cache.json?job=pages}
+ - |
+ if [ "$ARTIFACT_URL" != none ] && [ "$DISABLE_CACHE" == "no" ]
+ then
+ wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$ARTIFACT_URL" -O cache.json || true
+ fi
+ - |
+ if [ -s cache.json ] && [ "$DISABLE_CACHE" == "no" ]
+ then
+ echo Load cacheable data from cache.json
+ CACHE_ARGS="--cache cache.json"
+ fi
+ - ./bin/packaging-data-fetch-downstream
+ --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
+ --gitlab-server-url "${CI_SERVER_URL}"
+ --filter "${PROJECTS_NAMESPACE}/${FILTER_PACKAGES}"
+ --json packaging-data-downstream.json
+ ${CACHE_ARGS}
+ ${DEBUG:+--debug}
+ ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
+ artifacts:
+ when: always
+ paths:
+ - packaging-data-downstream.json
+ - ${LOG_TO_FILE}
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+packaging-data-fetch-sources-upstream:
+ stage: fetch
+ tags:
+ - lightweight
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ ca-certificates
+ python3-debian
+ python3-gitlab
+ python3-requests
+ python3-yaml
+ wget
+ script:
+ - CACHE_ARGS=""
+ - ARTIFACT_URL=${ARTIFACT_URL:-$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/$CI_DEFAULT_BRANCH/raw/packaging-cache.json?job=pages}
+ - |
+ if [ "$ARTIFACT_URL" != none ] && [ "$DISABLE_CACHE" == "no" ] && [ "$FILTER_ON_CACHE" == "yes" ]
+ then
+ wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$ARTIFACT_URL" -O cache.json || true
+ fi
+ - |
+ if [ -s cache.json ] && [ "$DISABLE_CACHE" == "no" ] && [ "$FILTER_ON_CACHE" == "yes" ]
+ then
+ echo Load cacheable data from cache.json
+ CACHE_ARGS="--cache cache.json"
+ fi
+ - ./bin/packaging-data-fetch-sources
+ --sources-definitions data/sources.yaml
+ --filter-packages "$FILTER_PACKAGES"
+ --json packaging-data-sources-upstream.json
+ ${CACHE_ARGS}
+ ${DEBUG:+--debug}
+ ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
+ artifacts:
+ when: always
+ paths:
+ - packaging-data-sources-upstream.json
+ - ${LOG_TO_FILE}
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+packaging-data-fetch-sources-published:
+ stage: fetch
+ tags:
+ - lightweight
+ before_script:
+ - *netrc-setup-snippet
+ - apt update && apt install -y --no-install-recommends
+ ca-certificates
+ python3-debian
+ python3-gitlab
+ python3-requests
+ python3-yaml
+ script:
+ - ./bin/packaging-data-fetch-sources
+ --sources-definitions data/channels.yaml
+ --sources-key channels
+ --output-key published
+ --filter-packages "$FILTER_PACKAGES"
+ --no-output-sources-definitions
+ --json packaging-data-sources-published.json
+ ${DEBUG:+--debug}
+ ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
+ artifacts:
+ when: always
+ paths:
+ - packaging-data-sources-published.json
+ - ${LOG_TO_FILE}
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+packaging-data-fetch-binaries-published:
+ stage: fetch
+ tags:
+ - lightweight
+ before_script:
+ - *netrc-setup-snippet
+ - apt update && apt install -y --no-install-recommends
+ ca-certificates
+ python3-debian
+ python3-gitlab
+ python3-requests
+ python3-yaml
+ script:
+ - ./bin/packaging-data-fetch-binaries
+ --sources-definitions data/channels.yaml
+ --sources-key channels
+ --output-key published
+ --filter-packages "$FILTER_PACKAGES"
+ --json packaging-data-binaries-published.json
+ ${DEBUG:+--debug}
+ ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
+ artifacts:
+ when: always
+ paths:
+ - packaging-data-binaries-published.json
+ - ${LOG_TO_FILE}
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+packaging-data-fetch-obs:
+ resource_group: obs
+ stage: fetch
+ timeout: 3h
+ tags:
+ - lightweight
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ ca-certificates
+ python3-debian
+ python3-gitlab
+ python3-m2crypto
+ python3-tenacity
+ python3-yaml
+ osc
+ script:
+ - ./bin/packaging-data-fetch-obs
+ --oscrc "$DASHBOARD_OSCRC"
+ --filter-packages "$FILTER_PACKAGES"
+ --json packaging-data-obs.json
+ ${DEBUG:+--debug}
+ ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
+ artifacts:
+ when: always
+ paths:
+ - packaging-data-obs.json
+ - ${LOG_TO_FILE}
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+storage-usage:
+ stage: fetch
+ tags:
+ - lightweight
+ cache:
+ - key:
+ files:
+ - storage_stats/Cargo.lock
+ paths:
+ - storage_stats/cargo/bin
+ - storage_stats/cargo/registry/index
+ - storage_stats/cargo/registry/cache
+ - storage_stats/cargo/git/db
+ - storage_stats/target
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ ca-certificates
+ cargo
+ libssl-dev
+ pkg-config
+ script:
+ - cd storage_stats
+ - export CARGO_HOME=$PWD/cargo
+ - test -n "$DEBUG" && RUST_LOG=debug
+ - cargo run --release --
+ --sources-definitions ../data/channels.yaml
+ --sources-key channels
+ --yaml ../storage.yaml
+ artifacts:
+ paths:
+ - storage.yaml
+ timeout: 3h
+ rules:
+ - if: $SKIP_STORAGE_USAGE == "yes"
+ when: never
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+packaging-check-settings:
+ resource_group: gitlab
+ stage: check
+ timeout: 15m
+ tags:
+ - lightweight
+ before_script:
+ - echo 'deb http://deb.debian.org/debian bookworm-backports main' > /etc/apt/sources.list.d/bookworm-backports.list
+ - apt update && apt install -y --no-install-recommends
+ ca-certificates
+ git
+ python3-debian
+ gitlab-rulez/bookworm-backports # required for `--output json`
+ script:
+ - GITLAB_RULES_GIT=$(echo $GITLAB_RULES_GIT | sed "s,/$CI_SERVER_HOST,/gitlab-ci-token:$CI_JOB_TOKEN@$CI_SERVER_HOST,")
+ - echo Retrieving rules $GITLAB_RULES_FILE from $GITLAB_RULES_GIT
+ - git clone --depth=1 "$GITLAB_RULES_GIT" rules
+ - ./bin/packaging-check-settings
+ --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
+ --gitlab-server-url "${CI_SERVER_URL}"
+ --filter "${PROJECTS_NAMESPACE}/${FILTER_PACKAGES}"
+ --rules "rules/$GITLAB_RULES_FILE"
+ --json packaging-check-settings.json
+ ${DEBUG:+--debug}
+ artifacts:
+ when: always
+ paths:
+ - packaging-check-settings.json
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+packaging-check-delta:
+ resource_group: gitlab
+ stage: check
+ timeout: 4h
+ tags:
+ - lightweight
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ git
+ python3-debian
+ python3-gitlab
+ python3-yaml
+ wget
+ script:
+ - CACHE_DELTA_ARGS=""
+ - ARTIFACT_URL=${ARTIFACT_URL:-$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/artifacts/$CI_DEFAULT_BRANCH/raw/packaging-cache.json?job=pages}
+ - |
+ if [ "$ARTIFACT_URL" != none ] && [ "$DISABLE_CACHE" == "no" ]
+ then
+ wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$ARTIFACT_URL" -O cache.json || true
+ fi
+ - |
+ if [ -s cache.json ] && [ "$DISABLE_CACHE" == "no" ]
+ then
+ echo Load cacheable data from cache.json
+ CACHE_DELTA_ARGS="--cache cache.json"
+ fi
+ - ./bin/json-merge
+ --input packaging-data-downstream.json
+ --input packaging-data-sources-upstream.json
+ --input packaging-data-sources-published.json
+ --input packaging-data-binaries-published.json
+ --input packaging-data-obs.json
+ --output packaging-data.json
+ - ./bin/packaging-check-delta
+ --projects packaging-data.json
+ --whitelists data/whitelists.yaml
+ --json packaging-check-delta.json
+ ${CACHE_DELTA_ARGS}
+ ${DEBUG:+--debug}
+ ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
+ artifacts:
+ when: always
+ paths:
+ - packaging-check-delta.json
+ - ${LOG_TO_FILE}
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+packaging-check-invariants:
+ stage: check
+ tags:
+ - lightweight
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ python3-debian
+ python3-gitlab
+ python3-yaml
+ script:
+ - ./bin/json-merge
+ --input packaging-data-downstream.json
+ --input packaging-data-sources-upstream.json
+ --input packaging-data-sources-published.json
+ --input packaging-data-binaries-published.json
+ --input packaging-data-obs.json
+ --output packaging-data.json
+ - ./bin/packaging-check-invariants
+ --projects packaging-data.json
+ --whitelists data/whitelists.yaml
+ --json packaging-checks.json
+ ${DEBUG:+--debug}
+ ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
+ artifacts:
+ when: always
+ paths:
+ - packaging-data.json
+ - packaging-checks.json
+ - ${LOG_TO_FILE}
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+packaging-updates:
+ stage: plan
+ tags:
+ - lightweight
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ devscripts
+ git
+ python3-debian
+ python3-gitlab
+ python3-yaml
+ script:
+ - ./bin/packaging-updates
+ --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
+ --projects packaging-data.json
+ --whitelists data/whitelists.yaml
+ --json packaging-updates.json
+ ${DEBUG:+--debug}
+ ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
+ artifacts:
+ when: always
+ paths:
+ - packaging-updates.json
+ - ${LOG_TO_FILE}
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+packaging-updates-upstream-linux:
+ stage: plan
+ tags:
+ - lightweight
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ ca-certificates
+ git
+ python3-debian
+ python3-gitlab
+ python3-yaml
+ script:
+ - ./bin/packaging-updates-upstream-linux
+ --projects packaging-data.json
+ --json packaging-updates-upstream-linux.json
+ ${DEBUG:+--debug}
+ ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
+ artifacts:
+ when: always
+ paths:
+ - packaging-updates-upstream-linux.json
+ - ${LOG_TO_FILE}
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+.render:
+ stage: render
+ tags:
+ - lightweight
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ python3-jinja2
+ python3-yaml
+ tree
+ script:
+ - STORAGE_DATA_ARGS=""
+ - |
+ if [ -e storage.yaml ]
+ then
+ echo Load storage data from storage.yaml
+ STORAGE_DATA_ARGS="--storage-data storage.yaml"
+ fi
+ - ./bin/json-merge
+ --input packaging-data.json
+ --input packaging-checks.json
+ --input packaging-check-delta.json
+ --input packaging-check-settings.json
+ --input packaging-updates.json
+ --input packaging-updates-upstream-linux.json
+ --output packaging.json
+ --output-cache packaging-cache.json
+ - ./bin/dashboard
+ --packaging-data packaging.json
+ ${STORAGE_DATA_ARGS}
+ --destdir public
+ --current-job-url "$CI_JOB_URL"
+ --current-pipeline-url "$CI_PIPELINE_URL"
+ --new-pipeline-url "$CI_PROJECT_URL/-/pipelines/new?ref=$CI_COMMIT_REF_NAME"
+ --os-website-url "${OS_WEBSITE}"
+ - ./bin/tsv
+ --data packaging.json
+ --destdir tsv/
+ - cd tsv && tree -H '.' -L 1 --noreport --charset utf-8 > index.html && cd ..
+ - mv ./tsv public/
+ artifacts:
+ when: always
+ expose_as: "Rendered dashboard"
+ paths:
+ - public/
+ - packaging-cache.json
+
+test:
+ extends:
+ - .render
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
+
+pages:
+ extends:
+ - .render
+ rules:
+ - if: $TRIGGER_FROM_JOB
+ when: never
+ - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_BRANCH
+ when: never
+ - if: $CI_PIPELINE_SOURCE != "merge_request_event"
diff --git a/.gitlab-ci/trigger-jobs.yml b/.gitlab-ci/trigger-jobs.yml
new file mode 100644
index 0000000..1ba32bb
--- /dev/null
+++ b/.gitlab-ci/trigger-jobs.yml
@@ -0,0 +1,70 @@
+trigger-updates:
+ interruptible: false
+ retry: 0
+ stage: trigger
+ tags:
+ - lightweight
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ python3-gitlab
+ python3-yaml
+ wget
+ script:
+ - |
+ if [ -n "$TRIGGER_FROM_JOB" ]
+ then
+ echo "Retrieving updatable packages from job $TRIGGER_FROM_JOB"
+ JOB_ID=${TRIGGER_FROM_JOB##*/}
+ JOB_API_URL=$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$JOB_ID/
+ wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$JOB_API_URL/artifacts/public/packaging.json"
+ else
+ cp public/packaging.json .
+ fi
+ - |
+ if [ -z "$TRIGGER_UPDATES" ]
+ then
+ echo 'Set TRIGGER_UPDATES by manually triggering the pipeline to actually initiate the updates'
+ echo '* use "*" to match everything'
+ echo '* use "dash" to only process the dash package'
+ echo 'If TRIGGER_UPDATES is left empty, do a dry run (this is the default).'
+ echo "For instance: $CI_PROJECT_URL/-/pipelines/new?var[TRIGGER_UPDATES]=*&ref=$CI_COMMIT_REF_NAME"
+ fi
+ - ./bin/trigger-updates
+ --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
+ --gitlab-server-url "${CI_SERVER_URL}"
+ --projects packaging.json
+ --whitelists data/whitelists.yaml
+ --filter "${TRIGGER_UPDATES}"
+ ${TRIGGER_SECURITY_UPDATES:+--only-security}
+ ${DEBUG:+--debug}
+ ${LOG_TO_FILE:+--log-to-file $LOG_TO_FILE}
+ artifacts:
+ when: always
+ paths:
+ - ${LOG_TO_FILE}
+
+trigger-gitlab-rulez:
+ interruptible: false
+ retry: 0
+ stage: trigger
+ tags:
+ - lightweight
+ rules:
+ - if: $TRIGGER_GITLAB_RULEZ == "apply"
+ before_script:
+ - apt update && apt install -y --no-install-recommends
+ gitlab-rulez
+ wget
+ script:
+ - GITLAB_RULES_GIT=$(echo $GITLAB_RULES_GIT | sed "s,/$CI_SERVER_HOST,/gitlab-ci-token:$CI_JOB_TOKEN@$CI_SERVER_HOST,")
+ - echo Retrieving rules $GITLAB_RULES_FILE from $GITLAB_RULES_GIT
+ - git clone --depth=1 "$GITLAB_RULES_GIT" rules
+ - gitlab-rulez
+ --gitlab-server-url "${CI_SERVER_URL}"
+ --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
+ "${TRIGGER_GITLAB_RULEZ}" "rules/$GITLAB_RULES_FILE"
+ --filter "${PROJECTS_NAMESPACE}/${FILTER_PACKAGES}" | tee log_gitlab_rulez.txt
+ artifacts:
+ when: always
+ paths:
+ - log_gitlab_rulez.txt
--
GitLab
From 0fbcd8757403fd72bf9487ef4baf22d64f07f38f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dylan=20A=C3=AFssi?= <dylan.aissi@collabora.com>
Date: Thu, 18 Jul 2024 14:37:46 +0200
Subject: [PATCH 2/2] Add variables to select which part (build dashboard or
trigger) to include in the pipeline
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Dylan Aïssi <dylan.aissi@collabora.com>
---
.gitlab-ci.yml | 18 ++++++++++++++++++
.gitlab-ci/trigger-jobs.yml | 7 ++++++-
templates/index.html.jinja2 | 6 +++---
3 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 61f89f2..8378960 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -78,10 +78,28 @@ variables:
options:
- "yes"
- "no"
+ INCLUDE_BUILD_DASHBOARD:
+ description: |
+ Include jobs building the dashboard.
+ value: "yes"
+ options:
+ - "yes"
+ - "no"
+ INCLUDE_TRIGGER_JOBS:
+ description: |
+ Include jobs triggering gitlab-rulez and trigger-updates.
+ value: "yes"
+ options:
+ - "yes"
+ - "no"
include:
- local: .gitlab-ci/dashboard.yml
+ rules:
+ - if: '$INCLUDE_BUILD_DASHBOARD == "yes" || $CI_PIPELINE_SOURCE == "merge_request_event"'
- local: .gitlab-ci/trigger-jobs.yml
+ rules:
+ - if: '$INCLUDE_TRIGGER_JOBS == "yes" || $CI_PIPELINE_SOURCE == "merge_request_event"'
stages:
- lint
diff --git a/.gitlab-ci/trigger-jobs.yml b/.gitlab-ci/trigger-jobs.yml
index 1ba32bb..9313165 100644
--- a/.gitlab-ci/trigger-jobs.yml
+++ b/.gitlab-ci/trigger-jobs.yml
@@ -17,8 +17,13 @@ trigger-updates:
JOB_ID=${TRIGGER_FROM_JOB##*/}
JOB_API_URL=$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$JOB_ID/
wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$JOB_API_URL/artifacts/public/packaging.json"
- else
+ elif [ -e public/packaging.json ]
+ then
+ echo "Using public/packaging.json to define updatable packages"
cp public/packaging.json .
+ else
+ echo "🛑 packaging.json is unavailable, please define TRIGGER_FROM_JOB or set INCLUDE_BUILD_DASHBOARD to yes"
+ exit 1
fi
- |
if [ -z "$TRIGGER_UPDATES" ]
diff --git a/templates/index.html.jinja2 b/templates/index.html.jinja2
index 83370e3..0355e1d 100644
--- a/templates/index.html.jinja2
+++ b/templates/index.html.jinja2
@@ -193,18 +193,18 @@
<ul class="list-inline mr-auto">
<a class="btn btn-primary {{'disabled' if not summary.wrong_settings_count }}"
title="Apply default settings to packages repositories"
- href="{{- meta.new_pipeline_url -}}&var[TRIGGER_GITLAB_RULEZ]=apply
+ href="{{- meta.new_pipeline_url -}}&var[TRIGGER_GITLAB_RULEZ]=apply&var[INCLUDE_BUILD_DASHBOARD]=no
">Run gitlab-rulez</a>
<a class="btn btn-primary {{'disabled' if not summary.total_updates_count }}"
title="Trigger pipelines importing updates"
- href="{{- meta.new_pipeline_url -}}&var[TRIGGER_UPDATES]=*
+ href="{{- meta.new_pipeline_url -}}&var[TRIGGER_UPDATES]=*&var[INCLUDE_BUILD_DASHBOARD]=no
{%- if meta.current_job_url -%}
&var[TRIGGER_FROM_JOB]={{- meta.current_job_url -}}
{%- endif -%}
">Update all</a>
<a class="btn btn-danger {{'disabled' if not summary.update_errors_count }}"
title="Trigger pipelines importing security updates"
- href="{{- meta.new_pipeline_url -}}&var[TRIGGER_UPDATES]=*&var[TRIGGER_SECURITY_UPDATES]=1
+ href="{{- meta.new_pipeline_url -}}&var[TRIGGER_UPDATES]=*&var[TRIGGER_SECURITY_UPDATES]=1&var[INCLUDE_BUILD_DASHBOARD]=no
{%- if meta.current_job_url -%}
&var[TRIGGER_FROM_JOB]={{- meta.current_job_url -}}
{%- endif -%}
--
GitLab