From 39b3d17da8f6c9561fd08e41fe4af1e67d09b2f6 Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Thu, 19 Nov 2026 17:41:47 +0000 Subject: [PATCH] jenkins: update `download-test.sh` script - No longer rsync to unencrypted, which has not been kept updated for some time so the script was finding old builds as the "most recent" ones for older release lines, and completely missing any releases for e.g. Node.js 26. - Use `index.tab` to find the most recent match instead of scraping the directory listing web page. - Use `expected_assets` and test all expected files for the release line the match belongs to (or the previous release line if it is a nightly from `main`) are present and validate their checksum to the entry in `SHASUMS256.txt`. Assisted-by: a closed-source coding agent Signed-off-by: Richard Lau --- jenkins/download-test.sh | 160 +++++++++++++++++++++++++++++---------- 1 file changed, 122 insertions(+), 38 deletions(-) diff --git a/jenkins/download-test.sh b/jenkins/download-test.sh index 9516261ae..2152b7fce 100755 --- a/jenkins/download-test.sh +++ b/jenkins/download-test.sh @@ -2,7 +2,7 @@ # Pass `NODE_VERSION` as an environment variable. Can be an exact or vague version. # NODE_VERSION=v8 ./download-test.sh --tap ./test.tap -while [ $# -gt 1 ]; do +while [ $# -gt 0 ]; do case "$1" in -t|--tap) @@ -16,56 +16,140 @@ while [ $# -gt 1 ]; do shift # Past option. done -if [ "$DOWNLOAD_LOCATION" == "Nightly" ]; then - downloadDir="nightly" -else - downloadDir="release" -fi +case $DOWNLOAD_LOCATION in + [Nn][Ii][Gg][Hh][Tt][Ll][Yy]) + downloadDir="nightly" + ;; + *) + downloadDir="release" + ;; +esac if [ -z "$NODE_VERSION" ]; then echo "NODE_VERSION is undefined! Please declare NODE_VERSION" exit 1 fi +indexUrl="https://nodejs.org/download/$downloadDir/index.tab" +indexTab=$(curl -sSf "$indexUrl") + +if [ -z "$indexTab" ]; then + echo "Failed to fetch index from $indexUrl" + exit 1 +fi + # If the provided NODE_VERSION is inexact, e.g. `v8`, get the most recent matching version. -LINK=`rsync rsync://unencrypted.nodejs.org/nodejs/$downloadDir/ | grep $NODE_VERSION | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1 | awk '{print $5}'` +VERSION=$(echo "$indexTab" | awk 'NR>1 {print $1}' | grep -E "${NODE_VERSION}" | head -n 1) + +if [ -z "$VERSION" ]; then + echo "No matches found for '$NODE_VERSION' in $indexUrl" + [ "$tapFile" ] && rm -rf "$tapFile" && echo "TAP version 13 +not ok 1 - find most recent downloads for ${NODE_VERSION} + --- + No matches found for '$NODE_VERSION' in $indexUrl + ... +1..1" > "$tapFile" + exit 1 +fi + +__dirname="$(CDPATH= cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPECTED_ASSETS_DIR="${__dirname}/../ansible/www-standalone/tools/promote/expected_assets" + +major=$(echo "$VERSION" | sed -E 's/^v([0-9]+).*/\1/') +LINE="v${major}.x" +EXPECTED_ASSETS_FILE="${EXPECTED_ASSETS_DIR}/${LINE}" + +if [ ! -f "$EXPECTED_ASSETS_FILE" ]; then + # Expected assets may not exist for nightlies from the `main` branch. + # Fall back to existing assets from the previous semver major line. + if [ "$downloadDir" = "nightly" ]; then + major=$((major - 1)) + PREVIOUS_LINE="v${major}.x" + EXPECTED_ASSETS_FILE="${EXPECTED_ASSETS_DIR}/${PREVIOUS_LINE}" + echo "No expected assets file found for ${LINE} in ${EXPECTED_ASSETS_DIR}, falling back to expected assets for ${PREVIOUS_LINE}" + LINE="${PREVIOUS_LINE}" + fi +fi +if [ ! -f "$EXPECTED_ASSETS_FILE" ]; then + echo "No expected assets file found for ${LINE} in ${EXPECTED_ASSETS_DIR}" + exit 1 +fi -if [ -z "$LINK" ]; then - echo "No matches found for '$NODE_VERSION' in https://unencrypted.nodejs.org/nodejs/$downloadDir/" +# Determine checksum tool +if command -v sha256sum >/dev/null 2>&1; then + shasum_cmd="sha256sum" +elif command -v shasum >/dev/null 2>&1; then + shasum_cmd="shasum -a 256" +else + echo "Neither sha256sum nor shasum is available" exit 1 fi # Remove old files -rm -rf v* -[ "$tapFile" ] && rm -rf "$tapFile" && echo "TAP version 13" > $tapFile +rm -rf "$VERSION" +[ "$tapFile" ] && rm -rf "$tapFile" && echo "TAP version 13" > "$tapFile" -# Scrape the download server -rsync -av rsync://unencrypted.nodejs.org/nodejs/"$downloadDir"/"$LINK" . +mkdir -p "$VERSION" +shasumsUrl="https://nodejs.org/download/$downloadDir/$VERSION/SHASUMS256.txt" +if ! curl -sSf "$shasumsUrl" -o "$VERSION/SHASUMS256.txt"; then + echo "Failed to download SHASUMS256.txt from $shasumsUrl" + exit 1 +fi testNumber=0 -cat "$LINK"/SHASUMS256.txt | { - while read line; do - let testNumber++ # Increment at the beginning - sha=$(echo "${line}" | awk '{print $1}') - file=$(echo "${line}" | awk '{print $2}') - echo "Checking shasum for $file" - calculatedSHA=$(shasum -a 256 "$LINK/$file" | awk '{print $1}') - if [ "$calculatedSHA" != "$sha" ]; then - echo "Error - SHASUMS256 does not match for $file" - echo "Expected - $sha" - echo "Found - $calculatedSHA" - if [ "$tapFile" ]; then - echo "not ok $testNumber SHASUMS256 does not match for $file - --- - Expected - $sha - Found - $remoteSHA - ... - " >> "$tapFile" - fi - else - echo "Pass - SHASUMS256 is correct for $file" - [ "$tapFile" ] && echo "ok $testNumber SHASUMS256 for $file is correct" >> "$tapFile" - fi - done +while IFS= read -r line || [ -n "$line" ]; do + # Trim whitespace + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + + # Skip empty lines, comments, and directory entries ending in / + [ -z "$line" ] && continue + [[ "$line" =~ ^# ]] && continue + [[ "$line" =~ /$ ]] && continue + + let testNumber++ + file="${line//\{VERSION\}/$VERSION}" + fileUrl="https://nodejs.org/download/$downloadDir/$VERSION/$file" + localFile="$VERSION/$file" + + mkdir -p "$(dirname "$localFile")" + + echo "Downloading $file" + if ! curl -sSf "$fileUrl" -o "$localFile"; then + echo "Error - Failed to download $file" + if [ "$tapFile" ]; then + echo "not ok $testNumber Failed to download $file + --- + URL - $fileUrl + ... +" >> "$tapFile" + fi + continue + fi + + sha=$(grep -E "[ *]${file}$" "$VERSION/SHASUMS256.txt" | awk '{print $1}') + if [ -n "$sha" ]; then + echo "Checking shasum for $file" + calculatedSHA=$($shasum_cmd "$localFile" | awk '{print $1}') + if [ "$calculatedSHA" != "$sha" ]; then + echo "Error - SHASUMS256 does not match for $file" + echo "Expected - $sha" + echo "Found - $calculatedSHA" + if [ "$tapFile" ]; then + echo "not ok $testNumber SHASUMS256 does not match for $file + --- + Expected - $sha + Found - $calculatedSHA + ... +" >> "$tapFile" + fi + else + echo "Pass - SHASUMS256 is correct for $file" + [ "$tapFile" ] && echo "ok $testNumber SHASUMS256 for $file is correct" >> "$tapFile" + fi + else + echo "Pass - Downloaded $file" + [ "$tapFile" ] && echo "ok $testNumber Downloaded $file" >> "$tapFile" + fi +done < "$EXPECTED_ASSETS_FILE" + [ "$tapFile" ] && echo "1..$testNumber" >> "$tapFile" -}