Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 6 additions & 80 deletions scripts/download_assets.sh
Original file line number Diff line number Diff line change
@@ -1,89 +1,15 @@
#!/usr/bin/env bash
# Downloads the assets of insight-docker.
# Downloads the assets of insight-docker into assets/.
#
# Downloads the assets needed to build and test the component into assets/, the only place from where its install
# scripts (src/) and its tests (test/) take them: building never accesses external systems. For each asset: if it is
# already in assets/, checks its SHA-256; if it is not, downloads it and then checks its SHA-256. The assets that are
# always the latest version of their source have no pinned SHA-256, so they are downloaded but not checked. Fails if
# any asset could not be downloaded or does not match its SHA-256. The external resources are listed in the
# documentation of bmde (docs/development/external-resources.md).
# The download logic is the one shared by every component of the BMDE (download_lib.sh, copied from bmde: see it for
# the details). The external resources are listed in the documentation of bmde (docs/development/external-resources.md).
#
# scripts/download_assets.sh [ASSET...] (every asset by default)
set -Eeuo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
ASSETS_DIR="${SCRIPT_DIR}/../assets"
SELECTED=" $* "
failed=0

# Downloads the URL $1 into the file $2
download() {
if command -v wget > /dev/null; then
wget -q "$1" -O "$2"
else
curl -fsSL "$1" -o "$2"
fi
}

# Obtains the source $1 into the file $2: a URL, or function:NAME to make it with the function NAME (given the file)
obtain() {
case "$1" in
function:*) "${1#function:}" "$2" ;;
*) download "$1" "$2" ;;
esac
}

# SHA-256 of the file $1 (replaced below for the sources of Insight)
# shellcheck disable=SC2329
checksum() {
sha256sum "$1" | cut -d ' ' -f 1
}

# asset NAME SHA256 SOURCE...: makes sure that assets/NAME is there and has the SHA-256 SHA256 (empty if it is not
# pinned). If it is not there, it is obtained from the first SOURCE that works. An asset without SOURCE is stored in
# the repository, so it is only checked
asset() {
local name="$1" sha256="$2" file="${ASSETS_DIR}/$1" source actual
shift 2
if [ "${SELECTED}" != " " ] && [[ "${SELECTED}" != *" ${name} "* ]]; then
return 0
fi
if [ -f "${file}" ]; then
echo "${name}: present"
else
mkdir -p "$(dirname "${file}")"
for source in "$@"; do
echo "${name}: downloading from ${source}"
if obtain "${source}" "${file}.part"; then
mv "${file}.part" "${file}"
break
fi
rm -f "${file}.part"
echo "${name}: could not be downloaded from ${source}" >&2
done
if [ ! -f "${file}" ] && [ "$#" -eq 0 ]; then
echo "${name}: ERROR, missing (it is stored in the repository: restore it with git)" >&2
failed=1
return 0
elif [ ! -f "${file}" ]; then
echo "${name}: ERROR, could not be downloaded" >&2
failed=1
return 0
fi
fi
if [ -z "${sha256}" ]; then
echo "${name}: not checked (latest version, without pinned SHA-256)"
return 0
fi
actual="$(checksum "${file}")"
if [ "${actual}" = "${sha256}" ]; then
echo "${name}: OK, the SHA-256 matches"
else
echo "${name}: ERROR, the SHA-256 does not match (expected ${sha256}, got ${actual}). Delete it to download it" \
"again" >&2
failed=1
fi
}
# shellcheck source=download_lib.sh
source "${SCRIPT_DIR}/download_lib.sh" "$@"

# Reconstructs the sources of Insight from the CVS repository of sourceware.org into the file $1 (~900 MB download)
# shellcheck disable=SC2329 # Called through a function: source of asset
Expand All @@ -93,7 +19,7 @@ fetch_insight_source() {

# The sources of Insight are checked by the SHA-256 of their files (of the list of the SHA-256 of every file, sorted by
# path) instead of the tarball, so the check does not depend on the versions of tar and xz that packed it
# shellcheck disable=SC2329 # Replaces the checksum function above
# shellcheck disable=SC2329 # Replaces the checksum function of download_lib.sh
checksum() {
if [[ "$1" != *-cvs.tar.xz* ]]; then
sha256sum "$1" | cut -d ' ' -f 1
Expand Down
91 changes: 91 additions & 0 deletions scripts/download_lib.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# shellcheck shell=bash
# shellcheck disable=SC2034 # failed is read by the script that sources the library
# Library of scripts/download_assets.sh of the components of the BMDE. The canonical copy is scripts/download_lib.sh of
# bmde, copied into scripts/ of each component by `make fmt` (and checked by `make lint`) so the download logic is the
# same in all of them: edit it there, not in a component.
#
# Downloads the assets needed to build and test the component into assets/, the only place from where its install
# scripts (src/) and its tests (test/) take them: building never accesses external systems. For each asset: if it is
# already in assets/, checks its SHA-256; if it is not, downloads it and then checks its SHA-256. The assets that are
# always the latest version of their source have no pinned SHA-256, so they are downloaded but not checked. The
# external resources are listed in the documentation of bmde (docs/development/external-resources.md).
#
# Source it with the arguments of the script (the names of the assets to download, every asset if there are none),
# declare each asset with `asset`, and end with `exit "${failed}"`, which fails if any asset could not be downloaded
# or does not match its SHA-256:
# source "${SCRIPT_DIR}/download_lib.sh" "$@"
# asset NAME SHA256 SOURCE...
# exit "${failed}"

ASSETS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/assets"
SELECTED=" $* "
failed=0

# Downloads the URL $1 into the file $2
download() {
if command -v wget > /dev/null; then
wget -q "$1" -O "$2"
else
curl -fsSL "$1" -o "$2"
fi
}

# Obtains the source $1 into the file $2: a URL, or function:NAME to make it with the function NAME (given the file)
obtain() {
case "$1" in
function:*) "${1#function:}" "$2" ;;
*) download "$1" "$2" ;;
esac
}

# SHA-256 of the file $1. A component can replace it after sourcing the library (e.g. insight-docker, whose sources
# are checked by the SHA-256 of the files they contain)
checksum() {
sha256sum "$1" | cut -d ' ' -f 1
}

# asset NAME SHA256 SOURCE...: makes sure that assets/NAME is there and has the SHA-256 SHA256 (empty if it is not
# pinned). If it is not there, it is obtained from the first SOURCE that works. An asset without SOURCE is stored in
# the repository, so it is only checked
asset() {
local name="$1" sha256="$2" file="${ASSETS_DIR}/$1" source actual
shift 2
if [ "${SELECTED}" != " " ] && [[ "${SELECTED}" != *" ${name} "* ]]; then
return 0
fi
if [ -f "${file}" ]; then
echo "${name}: present"
else
mkdir -p "$(dirname "${file}")"
for source in "$@"; do
echo "${name}: downloading from ${source}"
if obtain "${source}" "${file}.part"; then
mv "${file}.part" "${file}"
break
fi
rm -f "${file}.part"
echo "${name}: could not be downloaded from ${source}" >&2
done
if [ ! -f "${file}" ] && [ "$#" -eq 0 ]; then
echo "${name}: ERROR, missing (it is stored in the repository: restore it with git)" >&2
failed=1
return 0
elif [ ! -f "${file}" ]; then
echo "${name}: ERROR, could not be downloaded" >&2
failed=1
return 0
fi
fi
if [ -z "${sha256}" ]; then
echo "${name}: not checked (latest version, without pinned SHA-256)"
return 0
fi
actual="$(checksum "${file}")"
if [ "${actual}" = "${sha256}" ]; then
echo "${name}: OK, the SHA-256 matches"
else
echo "${name}: ERROR, the SHA-256 does not match (expected ${sha256}, got ${actual}). Delete it to download it" \
"again" >&2
failed=1
fi
}
Loading