sortix-mirror/build-aux/build-ports.sh

469 lines
16 KiB
Bash
Executable File

#!/bin/sh
set -e
make_dir_path_absolute() {
(cd "$1" && pwd)
}
has_command() {
which "$1" > /dev/null
}
# Determine what's supposed to happen.
OPERATION=build
if [ 1 -le $# ]; then OPERATION="$1"; fi
case "$OPERATION" in
download|extract|build) ;;
*) echo "$0: error: Invalid operation: $OPERATION" >&2
exit 1
esac
# Detect if the environment isn't set up properly.
if [ -z "$HOST" ]; then
echo "$0: error: You need to set \$HOST" >&2
exit 1
elif [ -z "$SYSROOT" ]; then
echo "$0: error: You need to set \$SYSROOT" >&2
exit 1
elif [ -z "$SORTIX_PORTS_DIR" ]; then
echo "$0: error: You need to set \$SORTIX_PORTS_DIR" >&2
exit 1
elif [ -z "$SORTIX_MIRROR_DIR" ]; then
echo "$0: error: You need to set \$SORTIX_MIRROR_DIR" >&2
exit 1
elif [ -z "$SORTIX_REPOSITORY_DIR" ]; then
echo "$0: error: You need to set \$SORTIX_REPOSITORY_DIR" >&2
exit 1
elif ! [ -d "$SORTIX_PORTS_DIR" ]; then
echo "Warning: No ports directory found, third party software will not be built"
exit 0
elif [ "$OPERATION" != build ]; then
: # tix isn't required for downloading and extracting
elif ! has_command tix-collection ||
! has_command tix-build ||
! has_command tix-install; then
echo "$0: error: You need to have installed Tix locally to compile ports." >&2
exit 1
fi
# Load the ports sets.
. "$(dirname -- "$0")/ports.conf"
# Create the mirror directory for downloaded archives.
mkdir -p "$SORTIX_MIRROR_DIR"
# Add the platform triplet to the binary repository path.
SORTIX_REPOSITORY_DIR="$SORTIX_REPOSITORY_DIR/$HOST"
mkdir -p "$SORTIX_REPOSITORY_DIR"
# Create the system root if absent.
if [ "$OPERATION" = build ]; then
mkdir -p "$SYSROOT"
fi
# Make paths absolute for later use.
if [ "$OPERATION" = build ]; then
SYSROOT=$(make_dir_path_absolute "$SYSROOT")
fi
SORTIX_MIRROR_DIR=$(make_dir_path_absolute "$SORTIX_MIRROR_DIR")
SORTIX_PORTS_DIR=$(make_dir_path_absolute "$SORTIX_PORTS_DIR")
SORTIX_REPOSITORY_DIR=$(make_dir_path_absolute "$SORTIX_REPOSITORY_DIR")
# Decide the optimization options with which the ports will be built.
if [ -z "${OPTLEVEL+x}" ]; then OPTLEVEL="-Os -s"; fi
if [ -z "${PORTS_OPTLEVEL+x}" ]; then PORTS_OPTLEVEL="$OPTLEVEL"; fi
if [ -z "${PORTS_CFLAGS+x}" ]; then PORTS_CFLAGS="$PORTS_OPTLEVEL"; fi
if [ -z "${PORTS_CXXFLAGS+x}" ]; then PORTS_CXXFLAGS="$PORTS_OPTLEVEL"; fi
if [ -z "${CFLAGS+x}" ]; then CFLAGS="$PORTS_CFLAGS"; fi
if [ -z "${CXXFLAGS+x}" ]; then CXXFLAGS="$PORTS_CXXFLAGS"; fi
CFLAGS="$CFLAGS -Werror=format -Wno-error=format-contains-nul -Werror=implicit-function-declaration"
CXXFLAGS="$CXXFLAGS -Werror=format -Wno-error=format-contains-nul"
export CFLAGS
export CXXFLAGS
# Initialize Tix package management in the system root if absent.
if [ "$OPERATION" = build ]; then
if [ !-e "$SYSROOT/tix/collection.conf" ]; then
tix-collection "$SYSROOT" create --platform=$HOST --prefix= --generation=2
fi
fi
# Detect all packages.
get_all_packages() {
for PACKAGE in $(ls "$SORTIX_PORTS_DIR"); do
if [ -f "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.info" ]; then
echo $PACKAGE
fi
done
}
# Detect which packages are available if not specified.
if [ -z "${PACKAGES+x}" ]; then
PACKAGES=$(get_all_packages)
fi
# Simply stop if there is no packages available.
if [ -z "$PACKAGES" ]; then
exit 0
fi
# Detect the build-time dependencies for a package.
get_package_dependencies_raw() {(
PACKAGE_DIR=$(echo $1 | grep -Eo '^[^\.]*')
if [ -f "$SORTIX_PORTS_DIR/$PACKAGE_DIR/$PACKAGE_DIR.info" ]; then
(BUILD_LIBRARIES= &&
. $SORTIX_PORTS_DIR/$PACKAGE_DIR/$PACKAGE_DIR.info &&
echo "$BUILD_LIBRARIES")
else
for set in $sets; do
if [ $set = $1 ]; then
echo $(eval echo \$set_$set)
fi
done
fi
)}
# Include the recursive dependencies if requested.
# TODO: This algorithm scales extremely poorly.
get_package_dependencies_recursive() {
for DEPENDENCY in $(get_package_dependencies_raw $1); do
WANT=false
if [ "$2" = "!!" ]; then
WANT=true
else
case "$DEPENDENCY" in
*"?") ;;
*) WANT=true ;;
esac
fi
if $WANT; then
DEPENDENCY=$(echo "$DEPENDENCY" | tr -d '?')
PACKAGE_DIR=$(echo "$DEPENDENCY" | grep -Eo '^[^\.]*')
# Optional dependencies might not exist yet.
if [ -f "$SORTIX_PORTS_DIR/$PACKAGE_DIR/$PACKAGE_DIR.info" ]; then
echo "$DEPENDENCY"
get_package_dependencies_recursive "$DEPENDENCY" "$2"
fi
fi
done
}
PACKAGES=$(for PACKAGE in $PACKAGES; do
RECURSION=$(echo "$PACKAGE" | grep -Eo '!*$')
PACKAGE=$(echo "$PACKAGE" | grep -Eo '^[^!]*')
# TODO: Better way of detecting a port is a set.
PACKAGE_DIR=$(echo "$PACKAGE" | grep -Eo '^[^\.]*')
if [ -f "$SORTIX_PORTS_DIR/$PACKAGE_DIR/$PACKAGE_DIR.info" ]; then
echo "$PACKAGE"
fi
if [ -n "$RECURSION" ]; then
get_package_dependencies_recursive "$PACKAGE" "$RECURSION"
fi
done | sort -u | tr '\n' ' ')
# Detect the build-time dependencies for a package with missing optional
# dependencies removed.
get_package_dependencies() {(
PRINTED_ANY=false
for DEPENDENCY in $(get_package_dependencies_raw $1); do
if [ "$DEPENDENCY" != "${DEPENDENCY%\?}" ]; then
DEPENDENCY="${DEPENDENCY%\?}"
FOUND=false
for PACKAGE in $PACKAGES; do
if [ "$PACKAGE" = "$DEPENDENCY" ]; then
FOUND=true
break
fi
done
if ! $FOUND; then
continue
fi
fi
if $PRINTED_ANY; then printf ' '; fi
printf "%s" "$DEPENDENCY"
PRINTED_ANY=true
done
if $PRINTED_ANY; then echo; fi
)}
# Decide the order the packages are built in according to their dependencies.
PACKAGES="$(echo "$PACKAGES" | tr ' ' '\n' | sort -R)"
DEPENDENCY_MAKEFILE=$(mktemp)
(for PACKAGE in $PACKAGES; do
echo "$PACKAGE: $(get_package_dependencies $PACKAGE)"
echo " @echo $PACKAGE"
done;
printf ".PHONY:"
for PACKAGE in $PACKAGES; do
printf " $PACKAGE"
done;
echo) > "$DEPENDENCY_MAKEFILE"
BUILD_LIST=$(unset MAKE;
unset MFLAGS;
unset MAKEFLAGS;
make -Bs -f "$DEPENDENCY_MAKEFILE" $PACKAGES)
rm -f "$DEPENDENCY_MAKEFILE"
PACKAGES="$BUILD_LIST"
# TODO: This adds another decompression and compression to the build time, this
# should be done as a tix post installation step. Also this might miss
# programs in unusual locations, so need a thorough search and strip.
strip_tix() {
DIR=$(mktemp -d)
tar -C "$DIR" -xf "$1"
$HOST-strip -d "$DIR/data/bin/"* 2>/dev/null || true
$HOST-strip -d "$DIR/data/lib/"* 2>/dev/null || true
$HOST-strip -d "$DIR/data/libexec"* 2>/dev/null || true
$HOST-strip -d "$DIR/data/libexec/git-core/"* 2>/dev/null || true
$HOST-strip -d "$DIR/data/sbin/"* 2>/dev/null || true
(cd "$DIR" && tar --numeric-owner --owner=0 --group=0 -cJf port.tar.tix.xz tix data)
cp "$DIR/port.tar.tix.xz" "$1"
rm -rf "$DIR"
}
announce() {
cat << EOF
================================================================================
==== $1
================================================================================
EOF
}
download_archive_from_url() {(
PACKAGE="$1"
ARCHIVE="$2"
URL="$3"
SHA256SUM="$4"
mkdir -p "$SORTIX_MIRROR_DIR/$PACKAGE"
if [ ! -e "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.sha256sum" ]; then
announce "Downloading $1: $ARCHIVE"
(if echo "$URL" | grep -Eq '^[a-z][a-z0-9.+-]*://'; then
if ! wget -O "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.untrusted" -- "$URL"; then
echo "warning: Failed to download $ARCHIVE from $URL" >&2
exit 1
fi
else
if ! cp -- "$URL" "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.untrusted"; then
echo "warning: Failed to copy $ARCHIVE from $URL" >&2
exit 1
fi
fi
if ! echo "$SHA256SUM $SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.untrusted" | sha256sum -c; then
sha256sum "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.untrusted"
echo "warning: Wrong sha256sum after downloading $ARCHIVE from $URL" >&2
exit 1
fi
mv "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.untrusted" "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE"
echo "$URL" > "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.url"
echo "$SHA256SUM $ARCHIVE" > "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.sha256sum.new"
mv "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.sha256sum.new" \
"$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.sha256sum"
) || (
rm -f "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.untrusted"
rm -f "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.url"
rm -f "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.sha56sum.new"
rm -f "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE.sha56sum"
exit 1
)
fi
)}
download_archive() {(
PACKAGE="$1"
ARCHIVE="$2"
UPSTREAM_SITE="$3"
UPSTREAM_ARCHIVE="$4"
SHA256SUM="$5"
if ! ([ -n "$SORTIX_PORTS_MIRROR" ] &&
download_archive_from_url "$PACKAGE" "$ARCHIVE" "$SORTIX_PORTS_MIRROR/$PACKAGE/$ARCHIVE" "$SHA256SUM") &&
! download_archive_from_url "$PACKAGE" "$ARCHIVE" "$UPSTREAM_SITE/$UPSTREAM_ARCHIVE" "$SHA256SUM"; then
echo "error: Failed to download $ARCHIVE with sha256sum $SHA256SUM" >&2
false
fi
)}
download_package() {(
PACKAGE="$1"
unset ARCHIVE
unset ARCHIVE_2
. "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.info"
if [ -n "$ARCHIVE" ]; then
download_archive "$PACKAGE" "$ARCHIVE" "$UPSTREAM_SITE" "$UPSTREAM_ARCHIVE" "$SHA256SUM"
if [ -n "$ARCHIVE_2" ]; then
download_archive "$PACKAGE" "$ARCHIVE_2" "$UPSTREAM_SITE_2" "$UPSTREAM_ARCHIVE_2" "$SHA256SUM_2"
fi
fi
)}
desired_version() {(
PACKAGE="$1"
unset DEVELOPMENT
unset ARCHIVE
unset ARCHIVE_2
. "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.info"
VERSION_STAMP="$NAME"
if [ -n "$ARCHIVE" ]; then
VERSION_STAMP="$VERSION_STAMP.$VERSION.$SHA256SUM"
if [ -n "$ARCHIVE_2" ]; then
VERSION_STAMP="$VERSION_STAMP.$VERSION_2.$SHA256SUM_2"
fi
fi
if [ -f "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.patch" ]; then
VERSION_STAMP="$VERSION_STAMP.$(cat "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.patch" | sha256sum | grep -Eo '^[^ ]*')"
fi
if [ -f "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.execpatch" ]; then
VERSION_STAMP="$VERSION_STAMP.$(cat "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.execpatch" | sha256sum | grep -Eo '^[^ ]*')"
fi
if [ -f "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.normalize" ]; then
VERSION_STAMP="$VERSION_STAMP.$(cat "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.normalize" | sha256sum | grep -Eo '^[^ ]*')"
fi
if [ "$DEVELOPMENT" = true ]; then
VERSION_STAMP="$VERSION_STAMP.development"
fi
echo "$VERSION_STAMP"
)}
diff_package() {(
# TODO: Only do this once the port is known to be cleaned.
cd "$SORTIX_PORTS_DIR/$1"
if [ -e "$1.normalized" ]; then
diff --no-dereference -Naur -- "$1.normalized" "$1" > "$1.patch" || true
if [ ! -s "$1.patch" ]; then rm "$1.patch"; fi
tix-execdiff -- "$1.normalized" "$1" > "$1.execpatch"
if [ ! -s "$1.execpatch" ]; then rm "$1.execpatch"; fi
# TODO: tix-rmpatch
fi
)}
extract_package() {(
PACKAGE="$1"
unset DEVELOPMENT
unset ARCHIVE
unset ARCHIVE_2
unset NEED_WRITABLE
. "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.info"
VERSION_STAMP="$(desired_version "$PACKAGE")"
if [ ! -e "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.version" ] ||
[ "$(cat "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.version")" != "$VERSION_STAMP" ] ||
[ ! -e "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.version" ]; then
if [ -e "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.version" ]; then
OLD_VERSION_STAMP="$(cat "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.version")"
case "$OLD_VERSION_STAMP" in
*.development)
if [ "$DEVELOPMENT" = true ]; then
echo "error: $PACKAGE: Refusing to delete port in development"
echo "error: $PACKAGE: .version is currently: $OLD_VERSION_STAMP"
echo "error: $PACKAGE: .version should be: $VERSION_STAMP"
exit 1
fi
esac
fi
echo "$VERSION_STAMP" > "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.version.new"
rm -rf "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE"
rm -rf "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.normalized"
mkdir "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE"
if [ "$DEVELOPMENT" = true ]; then
mkdir "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.normalized"
fi
if [ -n "$ARCHIVE" ]; then
announce "Extracting $1: $ARCHIVE"
tar -C "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE" -xf "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE" --strip-components=1
if [ "$DEVELOPMENT" = true ]; then
tar -C "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.normalized" -xf "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE" --strip-components=1
fi
if [ -n "$ARCHIVE_2" ]; then
announce "Extracting $1: $ARCHIVE_2"
tar -C "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE" -xf "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE_2" --strip-components=1
if [ "$DEVELOPMENT" = true ]; then
tar -C "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.normalized" -xf "$SORTIX_MIRROR_DIR/$PACKAGE/$ARCHIVE_2" --strip-components=1
fi
fi
fi
if [ -f "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.patch" ]; then
if [ "$DEVELOPMENT" = true ]; then
# TODO: .rej .orig etc.
patch -d "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE" -p1 < "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.patch" || true
# .orig files aren't useful unless part of the patch got rejected.
find "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE" \
-name '*.orig' \
-exec sh -c 'test ! -e "$(echo "$0" | sed -E "s,\\.orig$,.rej,")"' \
'{}' ';' -delete
else
patch -d "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE" -p1 < "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.patch"
fi
fi
if [ -f "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.execpatch" ]; then
tix-execpatch --directory "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE" "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.execpatch"
fi
if [ -f "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.normalize" ]; then
tix-rmpatch --directory "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE" "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.normalize"
if [ "$DEVELOPMENT" = true ]; then
tix-rmpatch --directory "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.normalized" "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.normalize"
fi
fi
if [ "$DEVELOPMENT" != true -a "$NEED_WRITABLE" != true ]; then
find "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE" '!' -type d -exec chmod -w '{}' +
fi
mv "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.version.new" "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.version"
if [ "$DEVELOPMENT" = true ]; then
REJECTS="$(find "$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE" \
-name '*.rej' -o -name '*.orig' | sort)"
if [ -n "$REJECTS" ]; then
echo "$REJECTS"
echo "error: $PACKAGE: The above patch hunks were rejected"
exit 1
fi
fi
fi
)}
# Build all the packages (if needed) and otherwise install them.
for PACKAGE in $PACKAGES; do
SOURCE_PACKAGE=$( (SOURCE_PACKAGE= &&
. $SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.info &&
echo "$SOURCE_PACKAGE") )
DEVELOPMENT=$( (DEVELOPMENT= &&
. $SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.info &&
echo "$DEVELOPMENT") )
# TODO: clean support.
# download
if [ -n "$SOURCE_PACKAGE" ]; then
download_package "$SOURCE_PACKAGE"
fi
download_package "$PACKAGE"
if [ "$OPERATION" = download ]; then continue; fi
# extract
if [ -n "$SOURCE_PACKAGE" ]; then
extract_package "$SOURCE_PACKAGE"
fi
extract_package "$PACKAGE"
if [ "$OPERATION" = extract ]; then continue; fi
# build
if [ ! -f "$SORTIX_REPOSITORY_DIR/$PACKAGE.tix.tar.xz" -o \
"$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE.version" -nt \
"$SORTIX_REPOSITORY_DIR/$PACKAGE.tix.tar.xz" ]; then
announce "Building $PACKAGE"
tix-build \
--sysroot="$SYSROOT" \
--host=$HOST \
--prefix= \
--destination="$SORTIX_REPOSITORY_DIR" \
--generation=2 \
${SOURCE_PACKAGE:+--source-package "$SORTIX_PORTS_DIR/$SOURCE_PACKAGE/$SOURCE_PACKAGE"} \
"$SORTIX_PORTS_DIR/$PACKAGE/$PACKAGE"
strip_tix "$SORTIX_REPOSITORY_DIR/$PACKAGE.tix.tar.xz"
if [ "$DEVELOPMENT" = true ]; then
# TODO: .rej .orig etc.
diff_package "$PACKAGE"
fi
fi
announce "Installing $PACKAGE"
tix-install \
--collection="$SYSROOT" \
--reinstall \
"$SORTIX_REPOSITORY_DIR/$PACKAGE.tix.tar.xz"
done