sortix-mirror/tix/tix-fetch

#!/bin/sh
# Copyright (c) 2017, 2021, 2023 Jonas 'Sortie' Termansen.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# tix-fetch
# Download operating system files.

set -e

boot=false
collection=""
continue=""
execpatch=false
initrd=false
insecure_downgrade_to_http=false
insecure_no_check_certificate=false
input_release_file=
input_release_sig_file=
input_sha256sum=
normalize=false
output=""
outputdir=""
output_release_file= # TODO: A better term for this?
output_release_sig_file= # TODO: A better term for this?
output_sha256sum=
patch=false
package=false
release=false
repository_metadata=false
sha256=false
sha256sum=false
source=false
source_full=false
sysroot=""
toolchain=false
url=false
url_main=false
url_main_release=false
url_mirror=false
url_mirror_release=false
url_release_sig=false
url_sha256sum=false
upgrade=false
# TODO: Option to select this default:
# TODO: This hides errors. Fix wget so it has a quiet, but errors, mode.
wget_options="-q --show-progress"

# TODO: Ability to get source code easily for gcc/binutils/libstdc++.

dashdash=
previous_option=
for argument do
  if test -n "$previous_option"; then
    eval $previous_option=\$argument
    previous_option=
    shift
    continue
  fi

  case $argument in
  *=?*) parameter=$(expr "X$argument" : '[^=]*=\(.*\)' || true) ;;
  *=)   parameter= ;;
  *)    parameter=yes ;;
  esac

  case $dashdash$argument in
  --) dashdash=yes ;;
  -c) continue="-c" ;;
  # TODO: Support -ofoo
  -o) previous_option=output ;;
  -O) previous_option=outputdir ;;
  -q) wget_options="-q" ;;
  -v) wget_options="-v" ;;
  --boot) boot=true ;;
  --collection=*) collection=$parameter ;;
  --collection) previous_option=collection ;;
  --continue) continue="--continue" ;;
  --download-non-verbose) wget_options="-nv" ;;
  --download-non-verbose) wget_options="-v" ;;
  --download-quiet) wget_options="-q" ;;
  --download-verbose) wget_options="-v" ;;
  --execpatch) execpatch=true ;;
  --initrd) initrd=true ;;
  --input-release-file=*) input_release_file=$parameter ;;
  --insecure-downgrade-to-http) insecure_downgrade_to_http=true ;;
  --insecure-no-check-certificate) insecure_no_check_certificate=true ;;
  --input-release-file) previous_option=input_release_file ;;
  --input-release-sig-file=*) input_release_sig_file=$parameter ;;
  --input-release-sig-file) previous_option=input_release_sig_file ;;
  --input-sha256sum=*) input_sha256sum=$parameter ;;
  --input-sha256sum) previous_option=input_sha256sum ;;
  --normalize) normalize=true ;;
  --nv) wget_options="-nv" ;;
  --outputdir=*) outputdir=$parameter ;;
  --outputdir) previous_option=outputdir ;;
  --output=*) output=$parameter ;;
  --output) previous_option=output ;;
  --output-release-file=*) output_release_file=$parameter ;;
  --output-release-file) previous_option=output_release_file ;;
  --output-release-sig-file=*) output_release_sig_file=$parameter ;;
  --output-release-sig-file) previous_option=output_release_sig_file ;;
  --output-sha256sum=*) output_sha256sum=$parameter ;;
  --output-sha256sum) previous_option=output_sha256sum ;;
  --output-upgrade-file=*) output_upgrade_file=$parameter ;;
  --output-upgrade-file) previous_option=output_upgrade_file ;;
  --package) package=true ;;
  --patch) patch=true ;;
  --repository-metadata) repository_metadata=true ;;
  --sha256) sha256=true ;;
  --sha256sum) sha256sum=true ;;
  --source-full) source_full=true ;;
  --source) source=true ;;
  --sysroot) previous_option=sysroot ;;
  --sysroot=*) sysroot=$parameter ;;
  --toolchain) toolchain=true ;;
  --upgrade) upgrade=true ;;
  --url) url=true ;;
  --url-main) url_main=true ;;
  --url-mirror) url_mirror=true ;;
  --url-main-release) url_main_release=true ;;
  --url-mirror-release) url_mirror_release=true ;;
  --url-release-sig) url_release_sig=true ;;
  --url-sha256sum) url_sha256sum=true ;;
  --wget-options) previous_option=wget_options ;;
  --wget-options=*) wget_options=$parameter ;;
  -*) echo "$0: unrecognized option $argument" >&2
      exit 1 ;;
  *) break ;;
  esac

  shift
done

if test -n "$previous_option"; then
  echo "$0: option '$argument' requires an argument" >&2
  exit 1
fi

# TODO: Mutually incompatible options.

conf() {
  sed -E -e 's/([a-zA-Z0-9_]+) *? *= */\U\1=/' \
         -e 's/=yes$/=true/' -e 's/no$/=false/' "$3" | \
  tix-vars -d "$2" - "$4"
}

tmpdir=$(mktemp -dt tix-fetch.XXXXXX)
trap 'rm -rf -- "$tmpdir"' EXIT HUP INT QUIT TERM

upgrade_conf="${collection%/}/etc/upgrade.conf"
CHANNEL=$(conf -d '' "$upgrade_conf" CHANNEL)
RELEASE_KEY=$(conf -d '' "$upgrade_conf" RELEASE_KEY)
RELEASE_SIG_URL=$(conf -d '' "$upgrade_conf" RELEASE_SIG_URL)
PREFERRED_MIRROR=$(conf -d '' "$upgrade_conf" PREFERRED_MIRROR)
FORCE_MIRROR=$(conf -d '' "$upgrade_conf" FORCE_MIRROR)
USER_AGENT="$(uname -s)/$(uname -r) ($(uname -m); $(uname -v))"

if $insecure_no_check_certificate; then
  echo "$0: warning: insecurely not checking https certificates" >&2
  wget_options="$wget_options --no-check-certificate"
fi

if $insecure_downgrade_to_http; then
  echo "$0: warning: insecurely downloading without https" >&2
  RELEASE_SIG_URL="$(echo "$RELEASE_SIG_URL" | sed -E 's,^https:,http:,')"
fi

if $url_release_sig; then
  printf "%s\n" "$RELEASE_SIG_URL"
  exit
fi

# HACK: Provide more useful errors when wget is silent:
do_wget() {
  (set +e
   wget "$@"
   status=$?
   set -e
   what=
   case $status in
   0) exit 0 ;;
   1) what="Generic error" ;;
   2) what="Parse error" ;;
   3) what="File I/O error" ;;
   4) what="Network I/O error" ;;
   5) what="Transport Layer Security verification failure" ;;
   6) what="Username/password failure" ;;
   7) what="Protocol error" ;;
   8) what="Error response" ;;
   *) what="Exit code $status" ;;
   esac
   echo "$0: $what when running: wget $@" >&2
   exit $status)
}

# Fetch signed release description.
download_release_sh() {
  (cd "$tmpdir" &&
   do_wget -U "$USER_AGENT" $wget_options -O release.sh.sig \
           -- "$RELEASE_SIG_URL")
  signify -Vq -p "$RELEASE_KEY" -em "$tmpdir/release.sh"
}

true > "$tmpdir/upgrade.sh"

if [ -z "$input_release_file" -a -z "$input_release_sig_file" ]; then
  download_release_sh
  # TODO: tix-vars's output is not quoted so it can be input again.
  tix-vars "$tmpdir/release.sh" | \
  grep -E '^UPGRADE_=' | \
  cat > "$tmpdir/upgrade.sh"
  UPGRADE_SIG_URL=$(tix-vars -d '' "$tmpdir/upgrade.sh" UPGRADE_SIG_URL)
  if $upgrade && [ -n "$UPGRADE_SIG_URL" ]; then
    RELEASE_SIG_URL="$UPGRADE_SIG_URL"
    RELEASE_KEY=$(tix-vars "$tmpdir/upgrade.sh" UPGRADE_KEY)
    download_release_sh
  fi
fi

if [ -n "$input_release_file" ]; then
  cp -T -- "$input_release_file" "$tmpdir/release.sh"
elif [ -n "$input_release_sig_file" ]; then
  signify -Vq -p "$RELEASE_KEY" -em "$tmpdir/release.sh"
fi

# Store the signed release file if requested.
if [ -n "$output_release_sig_file" ]; then
  cp -T -- "$tmpdir/release.sh.sig" "$output_release_sig_file"
fi

# Store the release file (without signature) if requested.
if [ -n "$output_release_file" ]; then
  cp -T -- "$tmpdir/release.sh" "$output_release_file"
fi

# Store the upgrade file if requested.
if [ -n "$output_upgrade_file" ]; then
  cp -T -- "$tmpdir/upgrade.sh" "$output_upgrade_file"
fi

# Load the release description.
# TODO: SECURITY: Protect against responding with older release.sh.

# TODO: DO NOT SUBMIT: Temporary compatibility.
MAIN=$(tix-vars -d '' "$tmpdir/release.sh" MAIN)
MASTER=$(tix-vars -d '' "$tmpdir/release.sh" MASTER)
if [ -z "$MAIN" ]; then
  MAIN="$MASTER"
fi

RELEASE=$(tix-vars "$tmpdir/release.sh" RELEASE)
MACHINE=$(tix-vars "$tmpdir/release.sh" MACHINE)
SHA256SUM_FILE=$(tix-vars -d sha256sum "$tmpdir/release.sh" SHA256SUM_FILE)
SHA256SUM_SHA256SUM=$(tix-vars "$tmpdir/release.sh" SHA256SUM_SHA256SUM)
MIRRORS=$(tix-vars -d '' "$tmpdir/release.sh" MIRRORS)

if $url_main; then
  printf "%s\n" "$MAIN"
  exit
elif $url_main_release; then
  printf "%s\n" "$MAIN/$RELEASE"
  exit
fi

# Default to the main mirror but switch to the preferred mirror if the release
# description knows about the mirror and believes it to be trustworthy.
MIRROR="$MAIN"
for POTENTIAL_MIRROR in $MIRRORS; do
  if [ "$POTENTIAL_MIRROR" = "$PREFERRED_MIRROR" ]; then
    MIRROR="$PREFERRED_MIRROR"
  fi
done
if [ -n "$PREFERRED_MIRROR" ] && [ "$MIRROR" != "$PREFERRED_MIRROR" ]; then
  if [ "$FORCE_MIRROR" = true ]; then
    MIRROR="$PREFERRED_MIRROR"
  else
    echo "$0: warning: ignoring unsupported mirror $PREFERRED_MIRROR" >&2
  fi
fi

# TODO: Make sure the distant future http downgrade is supported.
if $insecure_downgrade_to_http; then
  MIRROR="$(echo "$MIRROR" | sed -E 's,^https:,http:,')"
fi

if $url_mirror; then
  printf "%s\n" "$MIRROR"
  exit
elif $url_mirror_release; then
  printf "%s\n" "$MIRROR/$RELEASE"
  exit
fi

RELEASE_URL="$MIRROR/$RELEASE"

# Fetch sha256sum file and check its SHA256 hash with the release description.
if $url_sha256sum; then
  printf "%s\n" "$RELEASE_URL/$SHA256SUM_FILE"
  exit
fi
if [ -z "$input_sha256sum" ]; then
  # TODO: If the mirror doesn't work, try the main.
  (cd "$tmpdir" &&
   do_wget -U "$USER_AGENT" $wget_options -O sha256sum \
           -- "$RELEASE_URL/$SHA256SUM_FILE")
else
  cp -T -- "$input_sha256sum" "$tmpdir/sha256sum"
fi
# TODO: Check if upstream release description changed, if so, start over.
echo "$SHA256SUM_SHA256SUM  $tmpdir/sha256sum" | sha256sum -cq

# Store the sha256sum file if requested.
if [ -n "$output_sha256sum" ]; then
  cp -T -- "$tmpdir/sha256sum" "$output_sha256sum"
fi

escape_extended_regex() {
  printf "%s\n" "$1" | sed -E -e 's/[[$()*?\+.^{|}]/\\\0/g'
}

request() {
  REQUEST="$1"
  REQUESTDIR="$2"
  REQUESTFINAL="${3-$1}"
  FULLREQUEST="$REQUESTDIR$REQUEST"

  if $url; then
    printf '%s\n' "$RELEASE_URL/$FULLREQUEST"
    return
  fi

  if $sha256 || $sha256sum; then
    set +e # Don't fail if grep exits 1 (no match).
    # TODO: Should this be a checksum(1) feature to look up a hash?
    grep -E "^[0-9a-fA-F]{64}  $(escape_extended_regex "$FULLREQUEST")$" \
            "$tmpdir/sha256sum" > "$tmpdir/match"
    EXITCODE=$?
    set -e
    if [ 2 -le "$EXITCODE" ]; then (exit $EXITCODE); fi
    if $sha256 && [ -s "$tmpdir/match" ]; then
      grep -Eo '^[0-9a-fA-F]{64}' "$tmpdir/match"
    fi
    if $sha256sum && [ -s "$tmpdir/match" ]; then
      cat "$tmpdir/match"
    fi
    return
  fi

  # Decide the final location the file will end up.
  if [ -n "$output" ]; then
    FINAL="$output"
    OUTPUTDIR=$(dirname -- "$output")
  elif [ -n "$outputdir" ]; then
    FINAL="$outputdir/$REQUESTFINAL"
    OUTPUTDIR="$outputdir"
  else
    FINAL="$REQUESTFINAL"
    OUTPUTDIR=.
  fi

  # If a resumable download, store the file directly to the destination path.
  # Otherwise download to a temporary directory and move only to the final
  # location if the cryptographic check is passed.
  if [ -n "$continue" ]; then
    DOWNLOADDIR="$OUTPUTDIR"
    OUTPUT="$FINAL"
  else
    DOWNLOADDIR="$tmpdir/download"
    OUTPUT="$DOWNLOADDIR/$REQUEST"
    mkdir -p -- "$DOWNLOADDIR"
  fi

  # Fetch the file.
  # TODO: If the mirror doesn't work, try the main.
  (cd "$DOWNLOADDIR" &&
   mkdir -p -- "$(dirname -- "$REQUEST")" &&
   do_wget -U "$USER_AGENT" $wget_options $continue -O "$REQUEST" \
           -- "$RELEASE_URL/$FULLREQUEST")

  # Verify the cryptographic integrity of the fetched file.
  ABSOLUTE_OUTPUT=$(realpath -- "$OUTPUT")
  mkdir -p -- "$tmpdir/check"
  (cd "$tmpdir/check" &&
   mkdir -p -- "$(dirname -- "$FULLREQUEST")"
   ln -s -- "$ABSOLUTE_OUTPUT" "$FULLREQUEST"
   if ! sha256sum -q -C "$tmpdir/sha256sum" -- "$FULLREQUEST"; then
    # Don't leave behind a file that didn't pass a cryptographic check.
    if [ -n "$continue" ]; then
      # TODO: Check if upstream release description changed, if so, start over.
      echo "error: Deleting corrupted output file: $OUTPUT" 2>&1
      rm -f -- "$OUTPUT"
    fi
    exit 1
   fi)
  rm -rf -- "$tmpdir/check"

  # Move the file to the final destination if not already.
  if [ -z "$continue" ]; then
    if [ -z "$output" ]; then
      (cd "$OUTPUTDIR" && mkdir -p -- "$(dirname -- "$REQUESTFINAL")")
    fi
    cp -T -- "$OUTPUT" "$FINAL"
    rm -rf -- "$tmpdir/download"
  fi
}

if $release; then
  MIRRORS=$(tix-vars "$tmpdir/release.sh" BUILD_FILE)
  request "$BUILD_FILE" "" "$(basename -- "$BUILD_FILE")"
fi

# TODO: --source, --source-full
# TODO: --binutils, --gcc, --libstdc++

# Fetch each of the specified signed files from the mirror.
for REQUEST; do
  if $package; then
    REQUEST="$REQUEST.tix.tar.xz"
    REQUESTDIR="repository/$MACHINE-sortix/"
  elif $repository_metadata; then
    REQUESTDIR="repository/$MACHINE-sortix/"
  elif $boot; then
    REQUEST="$REQUEST"
    REQUESTDIR="$MACHINE/boot/"
  elif $initrd; then
    REQUEST="$REQUEST.tar.xz"
    REQUESTDIR="$MACHINE/boot/"
  elif $patch; then
    REQUEST="$REQUEST.patch"
    REQUESTDIR="patches/"
  elif $normalize; then
    REQUEST="$REQUEST.normalize"
    REQUESTDIR="patches/"
  elif $toolchain; then
    REQUESTDIR="toolchain/"
  else
    REQUESTDIR=""
  fi
  request "$REQUEST" "$REQUESTDIR"
done