mirror-to-github

#!/usr/bin/env bash

# --------------------------------------------------------------------------- #
# Mirror the SMS++ project from GitLab to the SMSpp-Project org on GitHub.    #
#                                                                             #
# Discovers every project in the GitLab `smspp` group (not just the ones      #
# in the umbrella) and, for each, idempotently:                               #
#   1. creates the matching public GitHub repo if it does not exist;          #
#   2. keeps the GitHub description in sync with the GitLab one;              #
#   3. configures a native GitLab push mirror to it (Settings ->              #
#      Repository -> Mirroring repositories) through the GitLab API;          #
#   4. disables the Wiki tab, which GitHub enables by default, everywhere     #
#      but on the umbrella, whose Wiki tab receives the pages of the GitLab   #
#      project wiki (a repository the native mirror does not cover).          #
# Re-running it is safe: existing repos and mirrors are detected and          #
# skipped, so a new GitLab project only requires running this again.          #
#                                                                             #
# Skipped: archived projects and `pysmspp` (itself a mirror of the            #
# GitHub-hosted pySMSpp).                                                     #
#                                                                             #
# Prerequisites:                                                              #
#   - gh (GitHub CLI) authenticated:  gh auth login -h github.com             #
#   - GITLAB_TOKEN : a GitLab Personal Access Token with the `api` scope      #
#     and Maintainer/Owner role on the projects, generated at:                #
#       https://gitlab.com/-/user_settings/personal_access_tokens             #
#   - GITHUB_MIRROR_PAT : a GitHub Personal Access Token (classic) with       #
#     the `repo` and `workflow` scopes; the latter lets the mirror            #
#     push the .github/workflows files, generated at:                         #
#       https://github.com/settings/tokens                                    #
#                                                                             #
# Usage:                                                                      #
#   GITLAB_TOKEN=glpat-xxx GITHUB_MIRROR_PAT=ghp_xxx bash mirror-to-github    #
#   DRY_RUN=1   ... bash mirror-to-github                          # preview  #
#   ROTATE=1    ... bash mirror-to-github  # rotate existing mirrors' token   #
#   WIKI_ONLY=1 ... bash mirror-to-github            # sync the wiki alone    #
#                                                                             #
#                                 Donato Meoli                                #
#                         Dipartimento di Informatica                         #
#                             Universita' di Pisa                             #
# --------------------------------------------------------------------------- #

set -uo pipefail

: "${GITLAB_TOKEN:?set GITLAB_TOKEN (GitLab PAT, scope api)}"
: "${GITHUB_MIRROR_PAT:?set GITHUB_MIRROR_PAT (GitHub PAT, scopes repo + workflow)}"
gh auth token >/dev/null 2>&1 || { echo "run 'gh auth login -h github.com' first" >&2; exit 1; }

API="https://gitlab.com/api/v4"
GROUP="smspp"
GH_ORG="SMSpp-Project"

# Emit, per non-archived group project, a TSV line:
#   <gitlab/path>\t<github-repo-name>\t<gitlab-description>
# GitHub repo names use the GitLab project name (CamelCase) for modules, with a
# few hand-mapped special cases; `pysmspp` is skipped (it is itself a mirror).
list_projects() {
  # with WIKI_ONLY=1 nothing is listed, so the run reduces to the wiki sync
  [ "${WIKI_ONLY:-0}" = "1" ] && return

  curl -sf -H "PRIVATE-TOKEN: $GITLAB_TOKEN" \
    "$API/groups/$GROUP/projects?per_page=100&include_subgroups=true&archived=false&visibility=public" \
  | jq -r '
      { smspp: "SMSpp", "smspp-project": "smspp-project",
        tests: "tests", tools: "tools", "vcpkg-registry": "vcpkg-registry" } as $special
      | .[]
      | select(.path != "pysmspp")
      | [ .path_with_namespace,
          ($special[.path] // .name),
          ((.description // "") | gsub("[\t|\n]"; " ") | gsub("(^ +| +$)"; "")) ]
      | @tsv'
}

while IFS=$'\t' read -r gl gh desc; do
  [ -n "$gl" ] || continue
  enc="${gl/\//%2F}"
  echo "== $gl -> $GH_ORG/$gh"

  # desired GitHub description = GitLab description + a mirror note
  if [ -n "$desc" ]; then
    want="$desc | mirror of https://gitlab.com/$gl"
  else
    want="Mirror of https://gitlab.com/$gl"
  fi

  # 1) GitHub repo: create if missing, keep its description in sync with GitLab
  if gh repo view "$GH_ORG/$gh" >/dev/null 2>&1; then
    have=$(gh repo view "$GH_ORG/$gh" --json description --jq '.description // ""')
    if [ "$have" = "$want" ]; then
      echo "   repo:   exists"
    elif [ "${DRY_RUN:-0}" = "1" ]; then
      echo "   repo:   would update description"
    else
      gh repo edit "$GH_ORG/$gh" --description "$want" >/dev/null \
        && echo "   repo:   description synced" || echo "   repo:   description sync FAILED"
    fi
  elif [ "${DRY_RUN:-0}" = "1" ]; then
    echo "   repo:   would create"
  else
    gh repo create "$GH_ORG/$gh" --public --description "$want" >/dev/null \
      && echo "   repo:   created" || { echo "   repo:   FAILED"; continue; }
  fi

  # 2) the project wiki lives on GitLab and is published on the GitHub side
  #    only through the umbrella (see sync_wiki below), so the Wiki tab that
  #    GitHub enables by default is turned off on every other mirror, where
  #    it would just offer an empty second place to write documentation
  if [ "$gh" != "smspp-project" ]; then
    if [ "$(gh repo view "$GH_ORG/$gh" --json hasWikiEnabled \
              --jq '.hasWikiEnabled' 2>/dev/null)" != "true" ]; then
      :
    elif [ "${DRY_RUN:-0}" = "1" ]; then
      echo "   wiki:   would disable"
    else
      gh repo edit "$GH_ORG/$gh" --enable-wiki=false >/dev/null \
        && echo "   wiki:   disabled" || echo "   wiki:   disable FAILED"
    fi
  fi

  # 3) GitLab push mirror to that GitHub repo. With ROTATE=1 an existing
  #    mirror is deleted and recreated, so its embedded GitHub token is
  #    refreshed (needed e.g. to grant the `workflow` scope).
  mid=$(curl -sf -H "PRIVATE-TOKEN: $GITLAB_TOKEN" "$API/projects/$enc/remote_mirrors" \
        | jq -r --arg u "github.com/${GH_ORG}/${gh}.git" \
            'first(.[] | select(.url | contains($u)) | .id) // empty')

  if [ -n "$mid" ] && [ "${ROTATE:-0}" != "1" ]; then
    echo "   mirror: exists"
  elif [ "${DRY_RUN:-0}" = "1" ]; then
    [ -n "$mid" ] && echo "   mirror: would rotate" || echo "   mirror: would create"
  else
    if [ -n "$mid" ]; then
      curl -sf -X DELETE -H "PRIVATE-TOKEN: $GITLAB_TOKEN" \
        "$API/projects/$enc/remote_mirrors/$mid" >/dev/null \
        || echo "   mirror: delete FAILED"
    fi
    curl -sf -X POST -H "PRIVATE-TOKEN: $GITLAB_TOKEN" \
      --data-urlencode "url=https://x-access-token:${GITHUB_MIRROR_PAT}@github.com/${GH_ORG}/${gh}.git" \
      --data "enabled=true" \
      --data "only_protected_branches=false" \
      --data "keep_divergent_refs=false" \
      "$API/projects/$enc/remote_mirrors" >/dev/null \
      && echo "   mirror: $([ -n "$mid" ] && echo rotated || echo created)" \
      || echo "   mirror: FAILED (Maintainer role required?)"
  fi

  # 4) trigger an immediate sync: a (re)created push mirror stays idle until
  #    the next push, so without this it would not propagate existing commits
  if [ "${DRY_RUN:-0}" != "1" ]; then
    sid=$(curl -sf -H "PRIVATE-TOKEN: $GITLAB_TOKEN" "$API/projects/$enc/remote_mirrors" \
          | jq -r --arg u "github.com/${GH_ORG}/${gh}.git" \
              'first(.[] | select(.url | contains($u)) | .id) // empty')
    [ -n "$sid" ] && curl -sf -o /dev/null -X POST -H "PRIVATE-TOKEN: $GITLAB_TOKEN" \
        "$API/projects/$enc/remote_mirrors/$sid/sync" \
        && echo "   mirror: sync triggered"
  fi
done < <(list_projects)

# The wiki of the umbrella is a repository of its own (<project>.wiki.git) that
# the native push mirror does not cover, and GitHub only links submodules that
# it can resolve to a github.com repository, so from the GitHub side the `wiki`
# gitlink is a dead entry. Mirroring the pages into the Wiki tab of the GitHub
# umbrella keeps them reachable there while GitLab stays the only place where
# they are written. GitHub materializes the wiki repository only after its
# first page has been created from the web UI, so until then the push is
# skipped with a note.
#
# The pages are copied verbatim but for the home, which gets a banner sending
# the reader back to GitLab: the Wiki tab of GitHub is writable, and whatever
# is written there is silently dropped by the next run.
add_banner() {
  local home="$1/Home.md"
  [ -f "$home" ] || { echo "   wiki:   no Home.md to bannerize"; return; }

  { printf '> ⚠️ This is a **read-only mirror** of the '
    printf '[SMS++ project wiki](https://gitlab.com/%s/%s), hosted on ' \
           "$GROUP" "smspp-project/-/wikis/home"
    printf '**GitLab**. Edits made here are overwritten by the next sync: '
    printf 'please edit the pages there.\n\n'
    cat "$home"
  } > "$home.tmp" && mv "$home.tmp" "$home"

  git -C "$1" -c user.name="SMS++ mirror" -c user.email="noreply@github.com" \
      commit --quiet -am "note that this copy of the wiki is read-only"
}

sync_wiki() {
  local src="https://oauth2:${GITLAB_TOKEN}@gitlab.com/${GROUP}/smspp-project.wiki.git"
  local dst="https://x-access-token:${GITHUB_MIRROR_PAT}@github.com/${GH_ORG}/smspp-project.wiki.git"
  local tmp

  echo "== $GROUP/smspp-project.wiki -> $GH_ORG/smspp-project (Wiki tab)"

  if ! git ls-remote --exit-code "$dst" >/dev/null 2>&1; then
    echo "   wiki:   not initialized on GitHub, create the first page at"
    echo "           https://github.com/$GH_ORG/smspp-project/wiki/_new"
    return
  fi
  if [ "${DRY_RUN:-0}" = "1" ]; then
    echo "   wiki:   would push"
    return
  fi

  tmp=$(mktemp -d) || { echo "   wiki:   FAILED (mktemp)"; return 1; }
  local out rc try

  # either side occasionally turns down a request that goes through moments
  # later, so one refusal is retried before calling the run failed
  rc=1
  for try in 1 2; do
    if out=$(git clone --quiet "$src" "$tmp/wiki" 2>&1); then
      rc=0
      break
    fi
    rm -rf "$tmp/wiki"
    [ "$try" = "1" ] && sleep 5
  done
  if [ "$rc" = "1" ]; then
    echo "   wiki:   clone FAILED twice"
    report "$out" "$GITLAB_TOKEN"
    rm -rf "$tmp"
    return 1
  fi

  add_banner "$tmp/wiki"

  rc=1
  for try in 1 2; do
    if out=$(git -C "$tmp/wiki" push --quiet --force "$dst" HEAD:master 2>&1); then
      echo "   wiki:   pushed"
      rc=0
      break
    fi
    [ "$try" = "1" ] && sleep 5
  done
  if [ "$rc" = "1" ]; then
    echo "   wiki:   push FAILED twice"
    report "$out" "$GITHUB_MIRROR_PAT"
  fi
  rm -rf "$tmp"
  return $rc
}

# echo what git said, with the token of the remote blanked out: it travels in
# the URL, which git repeats in most of its error messages
report() {
  [ -n "$1" ] || return
  echo "${1//"$2"/***}" | sed 's/^/           /'
}

echo
sync_wiki || wiki_rc=1

echo
echo "Done."

# a wiki left behind is worth a red run, the repositories keep themselves in
# sync through their own mirror but this copy only happens here
exit "${wiki_rc:-0}"