From 62e46cd53961c97ca4b68fa39834c79a7bead79a Mon Sep 17 00:00:00 2001 From: Sandro Wenzel Date: Sat, 21 Nov 2026 18:30:35 +0100 Subject: [PATCH] Keep the workflow printout in run-workflow-on-inputlist.sh This is fixing a bug where the printed workflow could be missing from the stdout of run-workflow-on-inputlist.sh. - The output was copied to stdout by a tail started after one second and killed when the workflow ended. - When the workflow had already ended, tail could be killed before printing anything. - tail now follows a sentinel process with --pid, which is stopped only after all processes are done. - The final wait for tail is limited to 10 s, with a warning if it has to be stopped. https://its.cern.ch/jira/browse/O2-7359 Co-Authored-By: Claude Opus 5.5 --- .../run-workflow-on-inputlist.sh | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/prodtests/full-system-test/run-workflow-on-inputlist.sh b/prodtests/full-system-test/run-workflow-on-inputlist.sh index 6622866afa745..ef4c9da75cda5 100755 --- a/prodtests/full-system-test/run-workflow-on-inputlist.sh +++ b/prodtests/full-system-test/run-workflow-on-inputlist.sh @@ -22,6 +22,7 @@ for i in EXTINPUT CTFINPUT RAWTFINPUT DIGITINPUT; do done NUM_PROCS=0 +PIDS_LOG= RETVAL=0 START_TIME=`date +%s` LOG_PREFIX="log_$(date +%Y%m%d-%H%M%S)_" @@ -79,13 +80,16 @@ start_process ${DPL_WORKFLOW_FROM_OUTSIDE:-$GEN_TOPO_MYDIR/dpl-workflow.sh} if [[ "0$4" != "00" ]]; then sleep 1 - tail -n 1000000 -f ${LOG_PREFIX}*.log & + # a sentinel that lives until all processes are done (or this script is gone); tail prints all logs and exits after it + ( while kill -0 $$ 2> /dev/null; do sleep 0.5; done ) > /dev/null 2>&1 & + PID_SENTINEL=$! + tail -n 1000000 -f -s 0.1 --pid=$PID_SENTINEL ${LOG_PREFIX}*.log & + PIDS_LOG="$PID_SENTINEL|$!" ln -sf ${LOG_PREFIX}${NUM_PROCS}.log latest.log - PID_LOG=$! fi TIMEOUT_PHASE=0 -while [[ `jobs -rl | grep -v " $PID_LOG Running" | wc -l` -ne 0 ]]; do +while [[ -n `jobs -rp | grep -v -x -E "${PIDS_LOG:-none}"` ]]; do sleep 1 if [[ -n $3 && $(date +%s) -ge $(($START_TIME + $TIMEOUT_PHASE * 20 + $3)) ]]; then RETVAL=1 @@ -97,8 +101,18 @@ while [[ `jobs -rl | grep -v " $PID_LOG Running" | wc -l` -ne 0 ]]; do fi done +# end the sentinel and give tail up to 10 s to print the rest of the logs if [[ "0$4" != "00" ]]; then - kill $PID_LOG + kill $PID_SENTINEL 2> /dev/null + for i in `seq 1 100`; do + [[ -z `jobs -rp` ]] && break + sleep 0.1 + done + if [[ -n `jobs -rp` ]]; then + echo "WARNING: stopping the log output of ${LOG_PREFIX}*.log, it may be incomplete" 1>&2 + kill `jobs -rp` 2> /dev/null + fi + wait ${PIDS_LOG//|/ } 2> /dev/null fi for i in `seq 1 $NUM_PROCS`; do