#!/usr/bin/env groovy

osname = 'apertis'

release = "18.12"

/* Determine whether to run uploads based on the prefix of the job name; in
 * case of apertis we expect the official jobs under apertis-<release>/ while
 * non-official onces can be in e.g. playground/ */
def production = env.JOB_NAME.startsWith("${osname}-")

docker_registry_name = 'docker-registry.apertis.org'
docker_image_name = "${docker_registry_name}/apertis/apertis-${release}-image-builder"

upload_host = "archive@images.apertis.org"
upload_root = "/srv/images/" + (production ? "public" : "test/${env.JOB_NAME}")
upload_dest = "${upload_host}:${upload_root}"
upload_credentials = '5a23cd79-e26d-41bf-9f91-d756c131b811'

image_url_prefix = "https://images.apertis.org" + (production ? "" : "/test/${env.JOB_NAME}")

ostree_path = "ostree/repo/"
ostree_pull_url = image_url_prefix + "/${ostree_path}"

test_repo_url = "git@gitlab.apertis.org:infrastructure/apertis-tests.git"
test_repo_credentials = 'df7b609b-df30-431d-a942-af263af80571'
test_repo_branch = 'master'
test_lava_credentials = 'apertis-lava-user'

demopack = "https://images.apertis.org/media/multimedia-demo.tar.gz"

sysroot_url_prefix = image_url_prefix + "/sysroot/"

def architectures = [
    amd64: [
        boards: ['uefi'],
        types: [
            minimal: [
                args: '',
                image: true,
                sysroot: false,
                ostree: true,
            ],
            target: [
                args: '',
                image: true,
                sysroot: false,
                ostree: true,
            ],
            sysroot: [
                args: '--scratchsize 10G',
                image: false,
                sysroot: true,
                ostree: false,
            ],
            sdk: [
                args: '--scratchsize 10G',
                boards: [ 'sdk' ],
                image: true,
                sysroot: false,
                ostree: false,
                requires: [
                    armhf: 'devroot',
                ]
            ],
            basesdk: [
                args: '--scratchsize 10G',
                boards: [ 'sdk' ],
                image: true,
                sysroot: false,
                ostree: false,
                requires: [
                    armhf: 'devroot',
                ]

            ]
        ]
    ],
    arm64: [
        boards: ['uboot'],
        types: [
            minimal: [
                args: '',
                image: true,
                sysroot: false,
                ostree: true,
            ],
            target: [
                args: '',
                image: true,
                sysroot: false,
                ostree: true,
            ],
            sysroot: [
                args: '--scratchsize 10G',
                image: false,
                sysroot: true,
                ostree: false,
            ]
        ]
    ],
    armhf: [
        boards: ['uboot', 'mx6qsabrelite'],
        types: [
            minimal: [
                args: '',
                image: true,
                sysroot: false,
                ostree: true
            ],
            sysroot: [
                args: '--scratchsize 10G',
                image: false,
                sysroot: true,
                ostree: false
            ],
            devroot: [
                args: '--scratchsize 10G',
                image: false,
                sysroot: false,
                ostree: false
            ]
        ]
    ]
]

properties([
    parameters([
        string(name: 'buildOnly', defaultValue: '', description: 'If set, only the selected images are built. Comma and slash separated, e.g. armhf/minimal, amd64/target', trim: true),
    ])
])

def buildOnlyList = params?.buildOnly.tokenize(/, ?/).collect { it.tokenize('/') }
buildOnlyList = buildOnlyList
  .groupBy{ it[0] }
  .collectEntries{ arch, values -> [arch, values.findResults { it[1] } ] }

def buildCandidates = architectures
if (buildOnlyList) {
    buildCandidates = buildCandidates
      // filter out architectures which won't be built
      .subMap(buildOnlyList.keySet())
      .collectEntries{ arch, architecture ->
        // filter out artifact types which won't be built
        types = architecture.types
        if (buildOnlyList[arch])
            types = architecture.types.subMap(buildOnlyList[arch])
        [arch, architecture + [
          types: types
        ]]
      }
}

def uploadDirectory(source, target) {
  sshagent(credentials: [ upload_credentials, ] ) {
    sh(script: """
      ssh -oStrictHostKeyChecking=no ${upload_host} mkdir -p ${upload_root}/${target}/
      rsync -e 'ssh -oStrictHostKeyChecking=no' -aP ${source} ${upload_dest}/${target}/
      """)
  }
}

def pushOstreeRepo(architecture, type, board) {
  def repo = "repo-${architecture}-${board}-${type}/"
  def branch = "${osname}/${release}/${architecture}-${board}/${type}"

  // push if the destination repository already exist, otherwise do a full repo upload
  // but use --ignore-existing to mitigate the chance of races
  sshagent(credentials: [ upload_credentials, ] ) {
    sh(script: """
    cd ${PIPELINE_VERSION}/${architecture}/${type}
    if ssh -oStrictHostKeyChecking=no ${upload_host} test -e ${upload_root}/${ostree_path}
    then
      ostree-push --repo ${repo} ${upload_dest}/${ostree_path} ${branch}
    else
      ostree remote --repo=${repo} delete origin # drop the remote before publishing
      ssh -oStrictHostKeyChecking=no ${upload_host} mkdir -p ${upload_root}/${ostree_path}/
      rsync -e 'ssh -oStrictHostKeyChecking=no' --ignore-existing -aP ${repo} ${upload_dest}/${ostree_path}
    fi""")
  }

  // Cleanup uploaded branch
  sh(script: """
    cd ${PIPELINE_VERSION}/${architecture}/${type}
    rm -rf ${repo}""")
}

def runTestsJobs(image_name, profile_name, version, submit = true) {

  if (!submit) {
    println "Skipping submitting tests jobs for ${profile_name} ${version}"
    return
  }

  dir ("apertis-tests") {
    git(url: test_repo_url,
        poll: false,
        credentialsId: test_repo_credentials,
        branch: test_repo_branch)
  }

  withCredentials([ file(credentialsId: test_lava_credentials, variable: 'lqaconfig'),
                    string(credentialsId: 'lava-phab-bridge-token', variable: 'token')]) {
    sh(script: """
    /usr/bin/lava-submit -c ${lqaconfig} \
      -g apertis-tests/templates/profiles.yaml \
      --profile ${profile_name} \
      --callback-secret ${token} \
      --callback-url https://lavaphabbridge.apertis.org/ \
      -t release:${release} \
      -t image_date:${version} \
      -t image_name:${image_name}""")
  }
}

def submitTests(architecture, type, board, ostree = false) {
  def image_name = imageName(architecture, type, board, ostree)

  def name = osname
  if(ostree){
    name = "${osname}_ostree"
  }
  def profile_name = "${name}-${type}-${architecture}-${board}"

  runTestsJobs (image_name, profile_name, env.PIPELINE_VERSION, true)
}

def buildOStree(architecture, type, board, debosarguments = "", repo = "repo") {
  def image_name = imageName(architecture, type, board, true)
  def branch = "${osname}/${release}/${architecture}-${board}/${type}"

  sh(script: """
      cd ${PIPELINE_VERSION}/${architecture}/${type}
      rm -rf ${repo}
      mkdir ${repo}
      ostree init --repo=${repo} --mode archive-z2
      ostree remote --repo=${repo} add --no-gpg-verify origin ${ostree_pull_url}
      http_code=\$(curl --location --silent -o /dev/null --head -w "%{http_code}" ${ostree_pull_url}/refs/heads/${branch})
      case \$http_code in
        200)
          ostree pull --repo=${repo} --depth=-1 --mirror --disable-fsync origin ${branch}
          ;;
        404)
          ;;
        *)
          echo "Error: Got HTTP \$http_code trying to fetch ${ostree_pull_url}/refs/heads/${branch}"
          exit 1
          ;;
      esac
      """)

  sh(script: """
      cd ${PIPELINE_VERSION}/${architecture}/${type}
      debos ${debosarguments} \
        --show-boot \
        -t architecture:${architecture} \
        -t type:$type \
        -t board:$board \
        -t suite:$release \
        -t ospack:ospack_${release}-${architecture}-${type}_${PIPELINE_VERSION} \
        -t image:${image_name} \
        -t message:${release}-${type}-${architecture}-${board}_${PIPELINE_VERSION} \
        -t ostree:${repo} \
        ${WORKSPACE}/${osname}-ostree-commit.yaml""")

  sh(script: """
      cd ${PIPELINE_VERSION}/${architecture}/${type}
      ostree --repo=${repo} static-delta generate \
        --empty \
        --to=${branch} \
        --inline \
        --min-fallback-size=1024 \
        --filename ${image_name}.delta""")
}

/** Generate the image name
 *
 * To have a single place for image name generation.
 *
 * @return string with the name
 */
def imageName(architecture, type, board, ostree = false){
  def name = osname
  if (ostree) {
    name = "${osname}_ostree"
  }
  return "${name}_${release}-${type}-${architecture}-${board}_${PIPELINE_VERSION}"
}

////////////////// High-level tasks //////////////////
def buildOSpack(architecture, type, debosarguments = "") {
    stage("${architecture} ${type} ospack") {
      sh(script: """
          cd ${PIPELINE_VERSION}/${architecture}/${type}
          debos ${debosarguments} \
            --show-boot \
            -t type:${type} \
            -t architecture:${architecture} \
            -t suite:${release} \
            -t timestamp:${PIPELINE_VERSION} \
            -t ospack:ospack_${release}-${architecture}-${type}_${PIPELINE_VERSION} \
            ${WORKSPACE}/${osname}-ospack-${type}.yaml""")
    }
}

def buildImage(architecture, type, board, debosarguments = "") {
    // Build image name here so it can easily be re-used by phases.
    def image_name = imageName(architecture, type, board, false)

    stage("${architecture} ${type} ${board} image") {
      sh(script: """
          cd ${PIPELINE_VERSION}/${architecture}/${type}
          debos ${debosarguments} \
            --show-boot \
            -t architecture:${architecture} \
            -t type:${type} \
            -t ospack:ospack_${release}-${architecture}-${type}_${PIPELINE_VERSION} \
            -t imageroot:${image_url_prefix}/daily/${release} \
            -t suite:$release \
            -t timestamp:${PIPELINE_VERSION} \
            -t image:${image_name} \
            -t demopack:${demopack} \
            ${WORKSPACE}/${osname}-image-${board}.yaml""")
    }
}

def buildOStreeImage(architecture, type, board, debosarguments = "") {
    def repo = "repo-${architecture}-${board}-${type}"
    def image_name = imageName(architecture, type, board, true)

    stage("${architecture} ${type} ${board} OStree image build") {
      buildOStree(architecture, type, board, debosarguments, repo)

      sh(script: """
          cd ${PIPELINE_VERSION}/${architecture}/${type}
          debos ${debosarguments} \
            --show-boot \
            -t architecture:${architecture} \
            -t type:$type \
            -t board:$board \
            -t suite:$release \
            -t ospack:ospack_${release}-${architecture}-${type}_${PIPELINE_VERSION} \
            -t image:${image_name} \
            -t demopack:${demopack} \
            -t message:${release}-${type}-${architecture}-${board}_${PIPELINE_VERSION} \
            -t ostree:${repo} \
            ${WORKSPACE}/apertis-ostree-image-${board}.yaml;""")
    }
}

def buildContainer(architecture, type, board, debosarguments = "") {
    def repo = "repo-${architecture}-${board}-${type}"
    buildOStree(architecture, type, board, debosarguments, repo)

    stage("${architecture} ${type} ${board} OStree pack") {
        sh(script: """
           cd ${PIPELINE_VERSION}/${architecture}/${type}
           debos ${debosarguments} \
             --show-boot \
             -t architecture:${architecture} \
             -t type:$type \
             -t suite:$release \
             -t repourl:${ostree_pull_url} \
             -t osname:${osname} \
             -t branch:${osname}/$release/${architecture}-${board}/${type} \
             -t ospack:${osname}_ostree_${release}-${type}-${architecture}-${board}_${PIPELINE_VERSION} \
             -t message:${release}-${type}-${architecture}-${type}-${board}_${PIPELINE_VERSION} \
             -t ostree:${repo} \
             ${WORKSPACE}/${osname}-ostree-pack.yaml""")
    }
}

def buildSysroot(architecture, type, debosarguments = "") {
    sysrootname = "sysroot-${osname}-${release}-${architecture}-${env.PIPELINE_VERSION}"
      stage("${architecture} sysroot tarball") {
        sh(script: """
            mkdir -p sysroot/${release}
            cd sysroot/${release}
            cp -l ${WORKSPACE}/${PIPELINE_VERSION}/${architecture}/${type}/ospack_${release}-${architecture}-${type}_${PIPELINE_VERSION}.tar.gz .
            debos ${debosarguments} \
              --show-boot \
              -t architecture:${architecture} \
              -t ospack:ospack_${release}-${architecture}-${type}_${PIPELINE_VERSION} \
              -t sysroot:${sysrootname} \
              ${WORKSPACE}/${osname}-sysroot.yaml; \
            rm ospack*""")

      // Generate sysroot metadata
      writeFile file: "sysroot/${release}/sysroot-${osname}-${release}-${architecture}", text: "version=${release} ${PIPELINE_VERSION}\nurl=${sysroot_url_prefix}${release}/${sysrootname}.tar.gz"
    }
}


/**
 * Build OSPack for architecture and start a parallel build of artifacts related
 * to target boards.
 *
 * @boards -- array with board names
 */
def buildImages(architecture, type, boards, debosarguments = "", image = true, sysroot = false, ostree = false, production = false) {
  return {
    node("docker-slave") {
      checkout scm
        docker.withRegistry("https://${docker_registry_name}") {
          buildenv = docker.image(docker_image_name)
            /* Pull explicitely to ensure we have the latest */
            buildenv.pull()

            buildenv.inside("--device=/dev/kvm") {
              stage("setup ${architecture} ${type}") {
                env.PIPELINE_VERSION = VersionNumber(versionNumberString: '${BUILD_DATE_FORMATTED,"yyyyMMdd"}.${BUILDS_TODAY_Z}')
                  sh ("env ; mkdir -p ${PIPELINE_VERSION}/${architecture}/${type}")
              }

              // Add successfully build artifacts here to know which ones we need to upload and test
              // Valid values atm:
              // - image-apt-${board} -- for apt-based images
              // - image-ostree-${board} -- for ostree-based images
              // - lxc-ostree -- for LXC tarball
              // - sysroot -- for sysroot tarball
              def buildStatus = [:]

              // The real work starts here
              try {
                // If that fails -- do not need to build the rest
                buildOSpack(architecture, type, debosarguments) 

                if (image) {
                  // Create apt-based images for all boards
                  for(String board: boards) {
                    try {
                      buildImage(architecture, type, board, debosarguments)
                      buildStatus["image-apt-${board}"] = true
                    } catch (e) {
                      // If image build failed -- do not fail other types but do not need to start tests for it
                      buildStatus["image-apt-${board}"] = false
                    }
                  }
                }


                if (ostree) {
                  // Create ostree-based images for all boards
                  for(String board: boards) {
                    try {
                      buildOStreeImage(architecture, type, board, debosarguments)
                      buildStatus["image-ostree-${board}"] = true
                    } catch (e) {
                      // If image build failed -- do not fail other types but do not need to start tests for it
                      buildStatus["image-ostree-${board}"] = false
                    }
                  }

                  /* Create ostree and tarball for container (board name = lxc) */
                  try {
                    buildContainer(architecture, type, "lxc", debosarguments)
                    buildStatus["lxc-ostree"] = true
                  } catch (e) {
                    // If image build failed -- do not fail other types but do not need to start tests for it
                    buildStatus["lxc-ostree"] = false
                  }
                }

                if (sysroot) {
                  // Create sysroot
                  try {
                    buildSysroot(architecture, type, debosarguments)
                    buildStatus["sysroot"] = true
                  } catch (e) {
                    // If image build failed -- do not fail other types but do not need to start tests for it
                    buildStatus["sysroot"] = false
                  }
                }

                // Mark the whole pipeline as failed in case of failure at any stage
                if (buildStatus.containsValue(false)) {
                      currentBuild.result = 'FAILURE'
                }

                // Upload artifacts
                stage("${architecture} ${type} upload") {
                  // Push OStree repos first to remove repository prior images upload
                  for(String board: boards) {
                    if(buildStatus["image-ostree-${board}"]) {
                      pushOstreeRepo(architecture, type, board)
                    }
                  }

                  if(buildStatus["lxc-ostree"]) {
                    pushOstreeRepo(architecture, type, "lxc")
                  }

                  // Upload all other artifacts like ospacks and images if any
                  uploadDirectory (env.PIPELINE_VERSION, "daily/${release}")

                  if(buildStatus["sysroot"]) {
                      uploadDirectory ("sysroot/${release}/*", "sysroot/${release}")
                  }
                }

                // This stage must be the last in pipeline
                // a failure to submit the tests would break the builds
                stage("Tests for ${architecture} ${type}") {
                  if (production) {
                    for(String board: boards) {
                      if(buildStatus["image-apt-${board}"]) {
                        submitTests(architecture, type, board, false)
                      }
                      if(buildStatus["image-ostree-${board}"]) {
                        submitTests(architecture, type, board, ostree)
                      }
                    }
                  } else {
                    println "Skipping submitting tests jobs for ${architecture} ${type}"
                  }
                }
              } finally {
                stage("Cleanup ${architecture} ${type}") {
                  deleteDir()
                }
              }
            }
        }
    }
  }
}

def first_pass = [:]
def second_pass = [:]

buildCandidates.each { name, arch ->
    arch.types.each { type, params ->
        /* merge the per-arch default with per-type params */
        def merged = [:] << arch << params

        if (!params.requires) {
            /* first, build all jobs which don’t have any dependencies, in parallel */
            first_pass << [("$name $type"):
                buildImages(name, type, merged.boards, merged.args, merged.image, merged.sysroot, merged.ostree, production)
            ]
        } else {
            /* second, build any jobs which depend on jobs from the first pass, also in parallel */
            second_pass << [("$name $type"):
                buildImages(name, type, merged.boards, merged.args, merged.image, merged.sysroot, merged.ostree, production)
            ]
        }
    }
}


parallel first_pass
parallel second_pass