diff --git a/jenkins/download-test.sh b/jenkins/download-test.sh index 9516261ae..2152b7fce 100755 --- a/jenkins/download-test.sh +++ b/jenkins/download-test.sh @@ -2,7 +2,7 @@ # Pass `NODE_VERSION` as an environment variable. Can be an exact or vague version. # NODE_VERSION=v8 ./download-test.sh --tap ./test.tap -while [ $# -gt 1 ]; do +while [ $# -gt 0 ]; do case "$1" in -t|--tap) @@ -16,56 +16,140 @@ while [ $# -gt 1 ]; do shift # Past option. done -if [ "$DOWNLOAD_LOCATION" == "Nightly" ]; then - downloadDir="nightly" -else - downloadDir="release" -fi +case $DOWNLOAD_LOCATION in + [Nn][Ii][Gg][Hh][Tt][Ll][Yy]) + downloadDir="nightly" + ;; + *) + downloadDir="release" + ;; +esac if [ -z "$NODE_VERSION" ]; then echo "NODE_VERSION is undefined! Please declare NODE_VERSION" exit 1 fi +indexUrl="https://nodejs.org/download/$downloadDir/index.tab" +indexTab=$(curl -sSf "$indexUrl") + +if [ -z "$indexTab" ]; then + echo "Failed to fetch index from $indexUrl" + exit 1 +fi + # If the provided NODE_VERSION is inexact, e.g. `v8`, get the most recent matching version. -LINK=`rsync rsync://unencrypted.nodejs.org/nodejs/$downloadDir/ | grep $NODE_VERSION | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1 | awk '{print $5}'` +VERSION=$(echo "$indexTab" | awk 'NR>1 {print $1}' | grep -E "${NODE_VERSION}" | head -n 1) + +if [ -z "$VERSION" ]; then + echo "No matches found for '$NODE_VERSION' in $indexUrl" + [ "$tapFile" ] && rm -rf "$tapFile" && echo "TAP version 13 +not ok 1 - find most recent downloads for ${NODE_VERSION} + --- + No matches found for '$NODE_VERSION' in $indexUrl + ... +1..1" > "$tapFile" + exit 1 +fi + +__dirname="$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPECTED_ASSETS_DIR="${__dirname}/../ansible/www-standalone/tools/promote/expected_assets" + +major=$(echo "$VERSION" | sed -E 's/^v([0-9]+).*/\1/') +LINE="v${major}.x" +EXPECTED_ASSETS_FILE="${EXPECTED_ASSETS_DIR}/${LINE}" + +if [ ! -f "$EXPECTED_ASSETS_FILE" ]; then + # Expected assets may not exist for nightlies from the `main` branch. + # Fall back to existing assets from the previous semver major line. + if [ "$downloadDir" = "nightly" ]; then + major=$((major - 1)) + PREVIOUS_LINE="v${major}.x" + EXPECTED_ASSETS_FILE="${EXPECTED_ASSETS_DIR}/${PREVIOUS_LINE}" + echo "No expected assets file found for ${LINE} in ${EXPECTED_ASSETS_DIR}, falling back to expected assets for ${PREVIOUS_LINE}" + LINE="${PREVIOUS_LINE}" + fi +fi +if [ ! -f "$EXPECTED_ASSETS_FILE" ]; then + echo "No expected assets file found for ${LINE} in ${EXPECTED_ASSETS_DIR}" + exit 1 +fi -if [ -z "$LINK" ]; then - echo "No matches found for '$NODE_VERSION' in https://unencrypted.nodejs.org/nodejs/$downloadDir/" +# Determine checksum tool +if command -v sha256sum >/dev/null 2>&1; then + shasum_cmd="sha256sum" +elif command -v shasum >/dev/null 2>&1; then + shasum_cmd="shasum -a 256" +else + echo "Neither sha256sum nor shasum is available" exit 1 fi # Remove old files -rm -rf v* -[ "$tapFile" ] && rm -rf "$tapFile" && echo "TAP version 13" > $tapFile +rm -rf "$VERSION" +[ "$tapFile" ] && rm -rf "$tapFile" && echo "TAP version 13" > "$tapFile" -# Scrape the download server -rsync -av rsync://unencrypted.nodejs.org/nodejs/"$downloadDir"/"$LINK" . +mkdir -p "$VERSION" +shasumsUrl="https://nodejs.org/download/$downloadDir/$VERSION/SHASUMS256.txt" +if ! curl -sSf "$shasumsUrl" -o "$VERSION/SHASUMS256.txt"; then + echo "Failed to download SHASUMS256.txt from $shasumsUrl" + exit 1 +fi testNumber=0 -cat "$LINK"/SHASUMS256.txt | { - while read line; do - let testNumber++ # Increment at the beginning - sha=$(echo "${line}" | awk '{print $1}') - file=$(echo "${line}" | awk '{print $2}') - echo "Checking shasum for $file" - calculatedSHA=$(shasum -a 256 "$LINK/$file" | awk '{print $1}') - if [ "$calculatedSHA" != "$sha" ]; then - echo "Error - SHASUMS256 does not match for $file" - echo "Expected - $sha" - echo "Found - $calculatedSHA" - if [ "$tapFile" ]; then - echo "not ok $testNumber SHASUMS256 does not match for $file - --- - Expected - $sha - Found - $remoteSHA - ... - " >> "$tapFile" - fi - else - echo "Pass - SHASUMS256 is correct for $file" - [ "$tapFile" ] && echo "ok $testNumber SHASUMS256 for $file is correct" >> "$tapFile" - fi - done +while IFS= read -r line || [ -n "$line" ]; do + # Trim whitespace + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + # Skip empty lines, comments, and directory entries ending in / + [ -z "$line" ] && continue + [[ "$line" =~ ^# ]] && continue + [[ "$line" =~ /$ ]] && continue + + let testNumber++ + file="${line//\{VERSION\}/$VERSION}" + fileUrl="https://nodejs.org/download/$downloadDir/$VERSION/$file" + localFile="$VERSION/$file" + + mkdir -p "$(dirname "$localFile")" + + echo "Downloading $file" + if ! curl -sSf "$fileUrl" -o "$localFile"; then + echo "Error - Failed to download $file" + if [ "$tapFile" ]; then + echo "not ok $testNumber Failed to download $file + --- + URL - $fileUrl + ... +" >> "$tapFile" + fi + continue + fi + + sha=$(grep -E "[ *]${file}$" "$VERSION/SHASUMS256.txt" | awk '{print $1}') + if [ -n "$sha" ]; then + echo "Checking shasum for $file" + calculatedSHA=$($shasum_cmd "$localFile" | awk '{print $1}') + if [ "$calculatedSHA" != "$sha" ]; then + echo "Error - SHASUMS256 does not match for $file" + echo "Expected - $sha" + echo "Found - $calculatedSHA" + if [ "$tapFile" ]; then + echo "not ok $testNumber SHASUMS256 does not match for $file + --- + Expected - $sha + Found - $calculatedSHA + ... +" >> "$tapFile" + fi + else + echo "Pass - SHASUMS256 is correct for $file" + [ "$tapFile" ] && echo "ok $testNumber SHASUMS256 for $file is correct" >> "$tapFile" + fi + else + echo "Pass - Downloaded $file" + [ "$tapFile" ] && echo "ok $testNumber Downloaded $file" >> "$tapFile" + fi +done < "$EXPECTED_ASSETS_FILE" + [ "$tapFile" ] && echo "1..$testNumber" >> "$tapFile" -}