.gitlab-ci.yml

default:
  interruptible: true
  image: debian:bullseye-slim

variables:
  DEBUG:
    description: Set to any non-empty value to enable debug output
    value: ""
  FILTER: 'pkg/*'
  TRIGGER_UPDATES:
    description: |
      Set to a glob of package names to automatically pull upstream updates on them.
      For instance use `*` to process all updates, `dash` to only process `pkg/dash`.
      Leave it empty to not trigger any update.
    value: ""

stages:
  - lint
  - fetch
  - check
  - plan
  - render
  - trigger

.netrc-setup-snippet: &netrc-setup-snippet
  - |
    if [ -f "$NETRC" ]
    then
      echo "Configuring credentials from the NETRC CI variable"
      cp "$NETRC" ~/.netrc
    fi

lint:
  stage: lint
  tags:
    - lightweight
  before_script:
    - apt update && apt install -y --no-install-recommends
        black
        flake8
        isort
  script:
    - isort bin/* localtest --check-only --diff
    - black bin/* localtest --check --diff
    - flake8 bin/* localtest
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule"
      allow_failure: true
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

localtest:
  stage: lint
  tags:
    - lightweight
  before_script:
    - apt update && apt install -y --no-install-recommends
        python3-yaml
  script:
    - ": do a test run of ./localtest, the tool used to run the pipeline during development"
    - ./localtest
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule"
      allow_failure: true
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

packaging-data-fetch-downstream:
  stage: fetch
  tags:
    - lightweight
  before_script:
    - apt update && apt install -y --no-install-recommends
        ca-certificates
        python3-debian
        python3-gitlab
        python3-yaml
  script:
    - ./bin/packaging-data-fetch-downstream
        --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
        --gitlab-server-url "${CI_SERVER_URL}"
        --filter "${FILTER}"
        --yaml packaging-data-downstream.yaml
        ${DEBUG:+--debug}
  artifacts:
    paths:
      - packaging-data-downstream.yaml
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

packaging-data-fetch-sources-upstream:
  stage: fetch
  tags:
    - lightweight
  before_script:
    - apt update && apt install -y --no-install-recommends
        ca-certificates
        python3-debian
        python3-gitlab
        python3-requests
        python3-yaml
  script:
    - ./bin/packaging-data-fetch-sources
        --sources-definitions data/sources.yaml
        --yaml packaging-data-sources-upstream.yaml
        ${DEBUG:+--debug}
  artifacts:
    paths:
      - packaging-data-sources-upstream.yaml
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

packaging-data-fetch-sources-published:
  stage: fetch
  tags:
    - lightweight
  before_script:
    - *netrc-setup-snippet
    - apt update && apt install -y --no-install-recommends
        ca-certificates
        python3-debian
        python3-gitlab
        python3-requests
        python3-yaml
  script:
    - ./bin/packaging-data-fetch-sources
        --sources-definitions data/channels.yaml
        --sources-key channels
        --output-key published
        --no-output-sources-definitions
        --yaml packaging-data-sources-published.yaml
        ${DEBUG:+--debug}
  artifacts:
    paths:
      - packaging-data-sources-published.yaml
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

packaging-data-fetch-binaries-published:
  stage: fetch
  tags:
    - lightweight
  before_script:
    - *netrc-setup-snippet
    - apt update && apt install -y --no-install-recommends
        ca-certificates
        python3-debian
        python3-gitlab
        python3-requests
        python3-yaml
  script:
    - ./bin/packaging-data-fetch-binaries
        --sources-definitions data/channels.yaml
        --sources-key channels
        --output-key published
        --yaml packaging-data-binaries-published.yaml
        ${DEBUG:+--debug}
  artifacts:
    paths:
      - packaging-data-binaries-published.yaml
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

packaging-data-fetch-obs:
  stage: fetch
  tags:
    - lightweight
  before_script:
    - apt update && apt install -y --no-install-recommends
        ca-certificates
        python3-debian
        python3-gitlab
        python3-m2crypto
        python3-yaml
        osc
  script:
    - ./bin/packaging-data-fetch-obs
        --oscrc "$DASHBOARD_OSCRC"
        --yaml packaging-data-obs.yaml
        ${DEBUG:+--debug}
  artifacts:
    paths:
      - packaging-data-obs.yaml
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

storage-usage:
  stage: fetch
  tags:
    - lightweight
  cache:
    - key:
        files:
          - storage_stats/Cargo.lock
      paths:
        - storage_stats/cargo/bin
        - storage_stats/cargo/registry/index
        - storage_stats/cargo/registry/cache
        - storage_stats/cargo/git/db
        - storage_stats/target
  before_script:
    - apt update && apt install -y --no-install-recommends
        ca-certificates
        cargo
        libssl-dev
        pkg-config
  script:
    - cd storage_stats
    - export CARGO_HOME=$PWD/cargo
    - test -n "$DEBUG" && RUST_LOG=debug
    - cargo run --release --
      --sources-definitions ../data/channels.yaml
      --sources-key channels
      --yaml ../storage.yaml
  artifacts:
    paths:
      - storage.yaml
  timeout: 1h 30m
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

packaging-check-invariants:
  stage: check
  before_script:
    - apt update && apt install -y --no-install-recommends
        python3-debian
        python3-gitlab
        python3-yaml
  script:
    - ./bin/yaml-merge
        --input packaging-data-downstream.yaml
        --input packaging-data-sources-upstream.yaml
        --input packaging-data-sources-published.yaml
        --input packaging-data-binaries-published.yaml
        --input packaging-data-obs.yaml
        --output packaging-data.yaml
    - ./bin/packaging-check-invariants
        --projects packaging-data.yaml
        --yaml packaging-checks.yaml
        ${DEBUG:+--debug}
  artifacts:
    paths:
      - packaging-data.yaml
      - packaging-checks.yaml
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

packaging-updates:
  stage: plan
  before_script:
    - apt update && apt install -y --no-install-recommends
        python3-debian
        python3-gitlab
        python3-yaml
  script:
    - ./bin/packaging-updates
        --projects packaging-data.yaml
        --yaml packaging-updates.yaml
        ${DEBUG:+--debug}
  artifacts:
    paths:
      - packaging-updates.yaml
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

packaging-updates-upstream-linux:
  stage: plan
  before_script:
    - apt update && apt install -y --no-install-recommends
        ca-certificates
        git
        python3-debian
        python3-gitlab
        python3-yaml
  script:
    - ./bin/packaging-updates-upstream-linux
        --projects packaging-data.yaml
        --yaml packaging-updates-upstream-linux.yaml
        ${DEBUG:+--debug}
  artifacts:
    paths:
      - packaging-updates-upstream-linux.yaml
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

.render:
  stage: render
  #tags:
  #  - lightweight
  before_script:
    - apt update && apt install -y --no-install-recommends
        python3-jinja2
        python3-yaml
        tree
  script:
    - ./bin/yaml-merge
        --input packaging-data.yaml
        --input packaging-checks.yaml
        --input packaging-updates.yaml
        --input packaging-updates-upstream-linux.yaml
        --output packaging.yaml
    - ./bin/dashboard
        --packaging-data packaging.yaml
        --storage-data storage.yaml
        --destdir public
        --current-job-url "$CI_JOB_URL"
        --current-pipeline-url "$CI_PIPELINE_URL"
        --new-pipeline-url "$CI_PROJECT_URL/-/pipelines/new?ref=$CI_COMMIT_REF_NAME"
    - ./bin/tsv
        --data packaging.yaml
        --destdir tsv/
    - cd tsv && tree -H '.' -L 1 --noreport --charset utf-8 > index.html && cd ..
    - mv ./tsv public/
  artifacts:
    paths:
      - public

test:
  extends:
    - .render
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

pages:
  extends:
    - .render
  rules:
    - if: $TRIGGER_FROM_JOB
      when: never
    - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_BRANCH
      when: never
    - if: $CI_PIPELINE_SOURCE != "merge_request_event"

trigger-updates:
  interruptible: false
  stage: trigger
  tags:
    - lightweight
  before_script:
    - apt update && apt install -y --no-install-recommends
        python3-gitlab
        python3-yaml
        wget
  script:
    - |
      if [ -n "$TRIGGER_FROM_JOB" ]
      then
        echo "Retrieving updatable packages from job $TRIGGER_FROM_JOB"
        JOB_ID=${TRIGGER_FROM_JOB##*/}
        JOB_API_URL=$CI_API_V4_URL/projects/$CI_PROJECT_ID/jobs/$JOB_ID/
        wget --header "JOB-TOKEN: $CI_JOB_TOKEN" "$JOB_API_URL/artifacts/public/packaging.yaml"
      else
        cp public/packaging.yaml .
      fi
    - |
      if [ -z "$TRIGGER_UPDATES" ]
      then
        echo 'Set TRIGGER_UPDATES by manually triggering the pipeline to actually initiate the updates'
        echo '* use "*" to match everything'
        echo '* use "dash" to only process the dash package'
        echo 'If TRIGGER_UPDATES is left empty, do a dry run (this is the default).'
        echo "For instance: $CI_PROJECT_URL/-/pipelines/new?var[TRIGGER_UPDATES]=*&ref=$CI_COMMIT_REF_NAME"
      fi
    - ./bin/trigger-updates
        --gitlab-api-token "${DASHBOARD_GITLAB_API_TOKEN}"
        --gitlab-server-url "${CI_SERVER_URL}"
        --projects packaging.yaml
        --filter "${TRIGGER_UPDATES}"