[actions] cache container images, verified against digests looked up live

Each container job now looks up the image's index, manifest,
and config digests from the registry,
keys an `actions/cache` entry on the config digest,
and only uses a cached image if its ID matches one of those digests;
otherwise it is discarded and the image is pulled (with retries) instead.
A poisoned or stale cache entry can therefore never be used,
and a cache hit needs only a couple of small manifest requests,
so this scales regardless of pull rate limits.

.github/scripts/docker-image.sh

@@ -0,0 +1,114 @@
+#!/bin/sh
+
+# Cache a container image as a tarball, keyed by digests looked up live from the registry: a cached
+# image is only used if it is exactly what the registry serves now, so a poisoned cache entry is never used.
+#
+# usage:
+#   sh .github/scripts/docker-image.sh resolve <image>
+#     prints `key=<config digest>` and `digests=<index, manifest, and config digests>`, for $GITHUB_OUTPUT
+#   DIGESTS='<digests>' sh .github/scripts/docker-image.sh ensure <image> <tarball>
+#     loads <tarball> if it holds one of $DIGESTS; otherwise pulls <image>, checks it, and saves it to <tarball>
+
+set -eu
+
+ACCEPT='application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json'
+
+sha256_of() {
+  if command -v sha256sum >/dev/null 2>&1; then
+    sha256sum "$1"
+  else
+    shasum -a 256 "$1"
+  fi | cut -d ' ' -f 1
+}
+
+# fetches a manifest into a file; digests are computed from its exact bytes
+registry_get() {
+  curl -fsSL --retry 5 --retry-all-errors -H "Accept: ${ACCEPT}" -o "$2" "$1"
+}
+
+resolve() {
+  IMAGE="$1"
+  HOST="${IMAGE%%/*}"
+  REST="${IMAGE#*/}"
+  REPO="${REST%:*}"
+  TAG="${REST##*:}"
+  OS="$(docker version --format '{{.Server.Os}}')"
+  ARCH="$(docker version --format '{{.Server.Arch}}')"
+
+  WORK="$(mktemp -d)"
+  trap 'rm -rf "${WORK}"' EXIT
+
+  registry_get "https://${HOST}/v2/${REPO}/manifests/${TAG}" "${WORK}/index.json"
+  INDEX_DIGEST="sha256:$(sha256_of "${WORK}/index.json")"
+
+  if jq -e '.manifests' "${WORK}/index.json" >/dev/null; then
+    MANIFEST_DIGEST="$(jq -r --arg os "${OS}" --arg arch "${ARCH}" '[.manifests[] | select(.platform.os == $os and .platform.architecture == $arch)][0].digest // empty' "${WORK}/index.json")"
+    if [ -z "${MANIFEST_DIGEST}" ]; then
+      echo "${IMAGE} has no ${OS}/${ARCH} image" >&2
+      exit 1
+    fi
+    registry_get "https://${HOST}/v2/${REPO}/manifests/${MANIFEST_DIGEST}" "${WORK}/manifest.json"
+    if [ "sha256:$(sha256_of "${WORK}/manifest.json")" != "${MANIFEST_DIGEST}" ]; then
+      echo "the ${OS}/${ARCH} manifest of ${IMAGE} does not match its digest" >&2
+      exit 1
+    fi
+  else
+    MANIFEST_DIGEST="${INDEX_DIGEST}"
+    cp "${WORK}/index.json" "${WORK}/manifest.json"
+  fi
+
+  CONFIG_DIGEST="$(jq -r '.config.digest' "${WORK}/manifest.json")"
+  echo "key=${CONFIG_DIGEST}"
+  echo "digests=${INDEX_DIGEST} ${MANIFEST_DIGEST} ${CONFIG_DIGEST}"
+}
+
+# an image's ID is its config digest (or, with the containerd image store, its manifest or index digest)
+image_matches() {
+  ID="$(docker image inspect --format '{{.Id}}' "$1" 2>/dev/null)" || return 1
+  case " ${DIGESTS} " in
+    *" ${ID} "*) return 0 ;;
+  esac
+  return 1
+}
+
+ensure() {
+  IMAGE="$1"
+  TARBALL="$2"
+
+  if [ -f "${TARBALL}" ]; then
+    if docker load -i "${TARBALL}" && image_matches "${IMAGE}"; then
+      echo "loaded ${IMAGE} from the cache"
+      return 0
+    fi
+    echo "::warning::the cached ${IMAGE} does not match the registry's digests; pulling it instead"
+    docker image rm -f "${IMAGE}" >/dev/null 2>&1 || true
+    rm -f "${TARBALL}"
+  fi
+
+  ATTEMPT=1
+  until docker pull "${IMAGE}"; do
+    if [ "${ATTEMPT}" -ge 5 ]; then
+      echo "docker pull ${IMAGE} failed after ${ATTEMPT} attempts" >&2
+      return 1
+    fi
+    echo "docker pull failed, attempt ${ATTEMPT}/5"
+    sleep $((ATTEMPT * 5))
+    ATTEMPT=$((ATTEMPT + 1))
+  done
+
+  # the tag can move between `resolve` and the pull; then this run uses the image, but does not cache it
+  if image_matches "${IMAGE}"; then
+    docker save -o "${TARBALL}" "${IMAGE}"
+  else
+    echo "::warning::the pulled ${IMAGE} no longer matches the digests looked up earlier; not caching it"
+  fi
+}
+
+case "${1-}" in
+  resolve) resolve "${2-}" ;;
+  ensure) ensure "${2-}" "${3-}" ;;
+  *)
+    echo 'usage: docker-image.sh resolve <image> | ensure <image> <tarball>' >&2
+    exit 2
+  ;;
+esac

.github/workflows/tests-alpine.yml

@@ -67,14 +67,22 @@ jobs:
           node-version: 'lts/*'
           skip-ls-check: true
       - run: npm ls urchin
+      # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's
+      # anonymous pull rate limits, and cache the image; a cached image is only used
+      # if it matches the digests the registry serves now
+      - name: 'Resolve the alpine:${{ matrix.alpine }} image digests'
+        id: image
+        run: sh .github/scripts/docker-image.sh resolve mirror.gcr.io/library/alpine:${{ matrix.alpine }} >> "$GITHUB_OUTPUT"
+      - uses: actions/cache@v6
+        with:
+          path: ${{ runner.temp }}/docker-image.tar
+          key: docker-image-${{ runner.arch }}-${{ steps.image.outputs.key }}
+      - name: 'Load or pull the alpine:${{ matrix.alpine }} image'
+        env:
+          DIGESTS: ${{ steps.image.outputs.digests }}
+        run: sh .github/scripts/docker-image.sh ensure mirror.gcr.io/library/alpine:${{ matrix.alpine }} "${RUNNER_TEMP}/docker-image.tar"
       - name: 'Run fast tests on Alpine ${{ matrix.alpine }} (${{ matrix.arch }})'
         run: |
-          # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's
-          # anonymous pull rate limits; retry to tolerate transient registry failures
-          for i in 1 2 3 4 5; do
-            docker pull mirror.gcr.io/library/alpine:${{ matrix.alpine }} && break
-            echo "docker pull failed, attempt $i/5"; sleep $((i * 5))
-          done
           docker run --rm \
             -v "${{ github.workspace }}:/workspace" \
             -w /workspace \
@@ -151,14 +159,22 @@ jobs:
       - uses: actions/checkout@v6
         with:
           submodules: false
+      # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's
+      # anonymous pull rate limits, and cache the image; a cached image is only used
+      # if it matches the digests the registry serves now
+      - name: 'Resolve the alpine:${{ matrix.alpine }} image digests'
+        id: image
+        run: sh .github/scripts/docker-image.sh resolve mirror.gcr.io/library/alpine:${{ matrix.alpine }} >> "$GITHUB_OUTPUT"
+      - uses: actions/cache@v6
+        with:
+          path: ${{ runner.temp }}/docker-image.tar
+          key: docker-image-${{ runner.arch }}-${{ steps.image.outputs.key }}
+      - name: 'Load or pull the alpine:${{ matrix.alpine }} image'
+        env:
+          DIGESTS: ${{ steps.image.outputs.digests }}
+        run: sh .github/scripts/docker-image.sh ensure mirror.gcr.io/library/alpine:${{ matrix.alpine }} "${RUNNER_TEMP}/docker-image.tar"
       - name: 'Install node ${{ matrix.node }} from a musl binary on Alpine ${{ matrix.alpine }}'
         run: |
-          # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's
-          # anonymous pull rate limits; retry to tolerate transient registry failures
-          for i in 1 2 3 4 5; do
-            docker pull mirror.gcr.io/library/alpine:${{ matrix.alpine }} && break
-            echo "docker pull failed, attempt $i/5"; sleep $((i * 5))
-          done
           docker run --rm \
             -v "${{ github.workspace }}:/workspace" \
             -w /workspace \

.github/workflows/tests-installation-node.yml

@@ -64,15 +64,22 @@ jobs:
           skip-ls-check: true
       - run: npm ls urchin
       - run: npx which urchin
+      # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's
+      # anonymous pull rate limits, and cache the image; a cached image is only used
+      # if it matches the digests the registry serves now
+      - name: Resolve the ubuntu:16.04 image digests
+        id: image
+        run: sh .github/scripts/docker-image.sh resolve mirror.gcr.io/library/ubuntu:16.04 >> "$GITHUB_OUTPUT"
+      - uses: actions/cache@v6
+        with:
+          path: ${{ runner.temp }}/docker-image.tar
+          key: docker-image-${{ runner.arch }}-${{ steps.image.outputs.key }}
+      - name: Load or pull the ubuntu:16.04 image
+        env:
+          DIGESTS: ${{ steps.image.outputs.digests }}
+        run: sh .github/scripts/docker-image.sh ensure mirror.gcr.io/library/ubuntu:16.04 "${RUNNER_TEMP}/docker-image.tar"
       - name: Run installation_node tests in container
         run: |
-          # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's
-          # anonymous pull rate limits; retry to tolerate transient registry failures
-          for i in 1 2 3 4 5; do
-            docker pull mirror.gcr.io/library/ubuntu:16.04 && break
-            echo "docker pull failed, attempt $i/5"
-            sleep $((i * 5))
-          done
           docker run --rm \
             -v "${{ github.workspace }}:/workspace" \
             -w /workspace \

.github/workflows/tests-xenial.yml

@@ -61,15 +61,22 @@ jobs:
           skip-ls-check: true
       - run: npm ls urchin
       - run: npx which urchin
+      # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's
+      # anonymous pull rate limits, and cache the image; a cached image is only used
+      # if it matches the digests the registry serves now
+      - name: Resolve the ubuntu:16.04 image digests
+        id: image
+        run: sh .github/scripts/docker-image.sh resolve mirror.gcr.io/library/ubuntu:16.04 >> "$GITHUB_OUTPUT"
+      - uses: actions/cache@v6
+        with:
+          path: ${{ runner.temp }}/docker-image.tar
+          key: docker-image-${{ runner.arch }}-${{ steps.image.outputs.key }}
+      - name: Load or pull the ubuntu:16.04 image
+        env:
+          DIGESTS: ${{ steps.image.outputs.digests }}
+        run: sh .github/scripts/docker-image.sh ensure mirror.gcr.io/library/ubuntu:16.04 "${RUNNER_TEMP}/docker-image.tar"
       - name: Run xenial tests in container
         run: |
-          # Pull through Google's Docker Hub mirror, which is not subject to Docker Hub's
-          # anonymous pull rate limits; retry to tolerate transient registry failures
-          for i in 1 2 3 4 5; do
-            docker pull mirror.gcr.io/library/ubuntu:16.04 && break
-            echo "docker pull failed, attempt $i/5"
-            sleep $((i * 5))
-          done
           docker run --rm \
             -v "${{ github.workspace }}:/workspace" \
             -w /workspace \