From caa2ef956599bc095b1472e137c0c4eefa4ada38 Mon Sep 17 00:00:00 2001 From: grvvy Date: Mon, 2 Mar 2026 17:20:29 -0700 Subject: [PATCH 1/3] increase hub command timeout duration, remove unnecessary build args --- Jenkinsfile | 33 ++++++++------------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 31d8b64c..83341b4b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -13,7 +13,6 @@ pipeline { reuseNode true args '''--group-add=20 --group-add=46 --device-cgroup-rule="c 189:* rmw" \ --device-cgroup-rule="c 166:* rmw" -v /dev/bus/usb:/dev/bus/usb \ - -e TESTER=0000000000000000325866e629a25623 -e EUT=RunningFromRAM \ -v /tmp/req_pipe:/tmp/req_pipe -v /tmp/res_pipe:/tmp/res_pipe''' } } @@ -23,7 +22,7 @@ pipeline { script { try { // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'SECONDS') { + timeout(time: 20, unit: 'SECONDS') { sh 'hubs all off' } } catch (err) { @@ -35,7 +34,7 @@ pipeline { script { try { // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'SECONDS') { + timeout(time: 20, unit: 'SECONDS') { sh 'hubs h1_eut reset' } } catch (err) { @@ -50,7 +49,7 @@ pipeline { script { try { // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'SECONDS') { + timeout(time: 20, unit: 'SECONDS') { sh 'hubs h1_tester h1_eut reset' } } catch (err) { @@ -70,7 +69,7 @@ pipeline { script { try { // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'SECONDS') { + timeout(time: 20, unit: 'SECONDS') { sh 'hubs h1_eut reset' } } catch (err) { @@ -90,7 +89,6 @@ pipeline { reuseNode true args '''--group-add=20 --group-add=46 --device-cgroup-rule="c 189:* rmw" \ --device-cgroup-rule="c 166:* rmw" -v /dev/bus/usb:/dev/bus/usb \ - -e TESTER=0000000000000000325866e629a25623 -e EUT=RunningFromRAM \ -v /tmp/req_pipe:/tmp/req_pipe -v /tmp/res_pipe:/tmp/res_pipe''' } } @@ -100,7 +98,7 @@ pipeline { script { try { // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'SECONDS') { + timeout(time: 20, unit: 'SECONDS') { sh 'hubs all off' } } catch (err) { @@ -112,7 +110,7 @@ pipeline { script { try { // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'SECONDS') { + timeout(time: 20, unit: 'SECONDS') { sh 'hubs hpro_eut reset' } } catch (err) { @@ -127,7 +125,7 @@ pipeline { script { try { // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'SECONDS') { + timeout(time: 20, unit: 'SECONDS') { sh 'hubs hpro_tester hpro_eut reset' } } catch (err) { @@ -144,7 +142,7 @@ pipeline { script { try { // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'SECONDS') { + timeout(time: 20, unit: 'SECONDS') { sh 'hubs all off' } } catch (err) { @@ -153,21 +151,6 @@ pipeline { } } } - retry(3) { - script { - try { - // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'SECONDS') { - sh 'hubs hpro_eut reset' - } - } catch (err) { - echo "USB hub port power server command timeout reached." - throw err - } - } - sh 'sleep 1s' - sh 'python3 ci-scripts/test_sgpio_debug.py' - } } } } From 0e04680435ae622ec5f29cb6addd12b0b7d527f7 Mon Sep 17 00:00:00 2001 From: grvvy Date: Mon, 2 Mar 2026 17:30:46 -0700 Subject: [PATCH 2/3] add timeouts to test script calls --- Jenkinsfile | 41 +++++++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 83341b4b..4088e28e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -58,11 +58,21 @@ pipeline { } } sh 'sleep 1s' - sh '''python3 ci-scripts/hackrf_test.py --ci --log log \ - --hostdir host/build/hackrf-tools/src/ \ - --fwupdate firmware/hackrf_usb/build/ \ - --tester 0000000000000000325866e629a25623 \ - --eut RunningFromRAM --unattended --rev r4''' + script { + try { + // Allow 10 seconds for the USB hub port power server to respond + timeout(time: 10, unit: 'MINUTES') { + sh '''python3 ci-scripts/hackrf_test.py --ci --log log \ + --hostdir host/build/hackrf-tools/src/ \ + --fwupdate firmware/hackrf_usb/build/ \ + --tester 0000000000000000325866e629a25623 \ + --eut RunningFromRAM --unattended --rev r4''' + } + } catch (err) { + echo "HackRF One Test timeout reached." + throw err + } + } sh 'hubs all off' } retry(3) { @@ -134,11 +144,22 @@ pipeline { } } sh 'sleep 1s' - sh '''python3 ci-scripts/hackrf_pro_test.py --ci --log log \ - --hostdir host/build/hackrf-tools/src \ - --fwupdate firmware/hackrf_usb/build \ - --tester 0000000000000000a06063c82338145f \ - --eut RunningFromRAM -p --rev r1.2''' + script { + try { + // Allow 10 seconds for the USB hub port power server to respond + timeout(time: 10, unit: 'MINUTES') { + sh '''python3 ci-scripts/hackrf_pro_test.py --ci --log log \ + --hostdir host/build/hackrf-tools/src \ + --fwupdate firmware/hackrf_usb/build \ + --tester 0000000000000000a06063c82338145f \ + --eut RunningFromRAM -p --rev r1.2''' + } + } catch (err) { + echo "HackRF Pro Test timeout reached." + throw err + } + } + script { try { // Allow 10 seconds for the USB hub port power server to respond From 5fa6907c645c7a71deb6b9e0ae19cac47b4f7480 Mon Sep 17 00:00:00 2001 From: grvvy Date: Mon, 2 Mar 2026 19:00:19 -0700 Subject: [PATCH 3/3] improve timeout handling --- Jenkinsfile | 180 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 140 insertions(+), 40 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4088e28e..2ee5b9e5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,3 +1,5 @@ +import org.jenkinsci.plugins.workflow.steps.FlowInterruptedException + pipeline { agent any stages { @@ -21,24 +23,44 @@ pipeline { sh './ci-scripts/build_h1_firmware.sh' script { try { - // Allow 10 seconds for the USB hub port power server to respond + // Allow 20 seconds for the USB hub port power server to respond timeout(time: 20, unit: 'SECONDS') { sh 'hubs all off' } - } catch (err) { - echo "USB hub port power server command timeout reached." + } catch (FlowInterruptedException err) { + // Check if the cause was specifically an exceeded timeout + def cause = err.getCauses().get(0) + if (cause instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { + echo "USB hub port power server command timeout reached." + throw err // Re-throw the exception to fail the build + } else { + echo "Build interrupted for another reason." + throw err // Re-throw the exception to fail the build + } + } catch (Exception err) { + echo "An unrelated error occurred: ${err.getMessage()}" throw err } } retry(3) { script { try { - // Allow 10 seconds for the USB hub port power server to respond + // Allow 20 seconds for the USB hub port power server to respond timeout(time: 20, unit: 'SECONDS') { sh 'hubs h1_eut reset' } - } catch (err) { - echo "USB hub port power server command timeout reached." + } catch (FlowInterruptedException err) { + // Check if the cause was specifically an exceeded timeout + def cause = err.getCauses().get(0) + if (cause instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { + echo "USB hub port power server command timeout reached." + throw err // Re-throw the exception to fail the build + } else { + echo "Build interrupted for another reason." + throw err // Re-throw the exception to fail the build + } + } catch (Exception err) { + echo "An unrelated error occurred: ${err.getMessage()}" throw err } } @@ -48,42 +70,92 @@ pipeline { retry(3) { script { try { - // Allow 10 seconds for the USB hub port power server to respond + // Allow 20 seconds for the USB hub port power server to respond timeout(time: 20, unit: 'SECONDS') { sh 'hubs h1_tester h1_eut reset' } - } catch (err) { - echo "USB hub port power server command timeout reached." + } catch (FlowInterruptedException err) { + // Check if the cause was specifically an exceeded timeout + def cause = err.getCauses().get(0) + if (cause instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { + echo "USB hub port power server command timeout reached." + throw err // Re-throw the exception to fail the build + } else { + echo "Build interrupted for another reason." + throw err // Re-throw the exception to fail the build + } + } catch (Exception err) { + echo "An unrelated error occurred: ${err.getMessage()}" throw err } } sh 'sleep 1s' script { try { - // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'MINUTES') { + // Allow 5 minutes for the test to run + timeout(time: 5, unit: 'MINUTES') { sh '''python3 ci-scripts/hackrf_test.py --ci --log log \ --hostdir host/build/hackrf-tools/src/ \ --fwupdate firmware/hackrf_usb/build/ \ --tester 0000000000000000325866e629a25623 \ --eut RunningFromRAM --unattended --rev r4''' } - } catch (err) { - echo "HackRF One Test timeout reached." + } catch (FlowInterruptedException err) { + // Check if the cause was specifically an exceeded timeout + def cause = err.getCauses().get(0) + if (cause instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { + echo "HackRF One Test timeout limit reached." + throw err // Re-throw the exception to fail the build + } else { + echo "Build interrupted for another reason." + throw err // Re-throw the exception to fail the build + } + } catch (Exception err) { + echo "An unrelated error occurred: ${err.getMessage()}" + throw err + } + } + script { + try { + // Allow 20 seconds for the USB hub port power server to respond + timeout(time: 20, unit: 'SECONDS') { + sh 'hubs all off' + } + } catch (FlowInterruptedException err) { + // Check if the cause was specifically an exceeded timeout + def cause = err.getCauses().get(0) + if (cause instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { + echo "USB hub port power server command timeout reached." + throw err // Re-throw the exception to fail the build + } else { + echo "Build interrupted for another reason." + throw err // Re-throw the exception to fail the build + } + } catch (Exception err) { + echo "An unrelated error occurred: ${err.getMessage()}" throw err } } - sh 'hubs all off' } retry(3) { script { try { - // Allow 10 seconds for the USB hub port power server to respond + // Allow 20 seconds for the USB hub port power server to respond timeout(time: 20, unit: 'SECONDS') { sh 'hubs h1_eut reset' } - } catch (err) { - echo "USB hub port power server command timeout reached." + } catch (FlowInterruptedException err) { + // Check if the cause was specifically an exceeded timeout + def cause = err.getCauses().get(0) + if (cause instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { + echo "USB hub port power server command timeout reached." + throw err // Re-throw the exception to fail the build + } else { + echo "Build interrupted for another reason." + throw err // Re-throw the exception to fail the build + } + } catch (Exception err) { + echo "An unrelated error occurred: ${err.getMessage()}" throw err } } @@ -107,24 +179,44 @@ pipeline { sh './ci-scripts/build_hpro_firmware.sh' script { try { - // Allow 10 seconds for the USB hub port power server to respond + // Allow 20 seconds for the USB hub port power server to respond timeout(time: 20, unit: 'SECONDS') { sh 'hubs all off' } - } catch (err) { - echo "USB hub port power server command timeout reached." + } catch (FlowInterruptedException err) { + // Check if the cause was specifically an exceeded timeout + def cause = err.getCauses().get(0) + if (cause instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { + echo "USB hub port power server command timeout reached." + throw err // Re-throw the exception to fail the build + } else { + echo "Build interrupted for another reason." + throw err // Re-throw the exception to fail the build + } + } catch (Exception err) { + echo "An unrelated error occurred: ${err.getMessage()}" throw err } } retry(3) { script { try { - // Allow 10 seconds for the USB hub port power server to respond + // Allow 20 seconds for the USB hub port power server to respond timeout(time: 20, unit: 'SECONDS') { sh 'hubs hpro_eut reset' } - } catch (err) { - echo "USB hub port power server command timeout reached." + } catch (FlowInterruptedException err) { + // Check if the cause was specifically an exceeded timeout + def cause = err.getCauses().get(0) + if (cause instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { + echo "USB hub port power server command timeout reached." + throw err // Re-throw the exception to fail the build + } else { + echo "Build interrupted for another reason." + throw err // Re-throw the exception to fail the build + } + } catch (Exception err) { + echo "An unrelated error occurred: ${err.getMessage()}" throw err } } @@ -138,36 +230,44 @@ pipeline { timeout(time: 20, unit: 'SECONDS') { sh 'hubs hpro_tester hpro_eut reset' } - } catch (err) { - echo "USB hub port power server command timeout reached." + } catch (FlowInterruptedException err) { + // Check if the cause was specifically an exceeded timeout + def cause = err.getCauses().get(0) + if (cause instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { + echo "USB hub port power server command timeout reached." + throw err // Re-throw the exception to fail the build + } else { + echo "Build interrupted for another reason." + throw err // Re-throw the exception to fail the build + } + } catch (Exception err) { + echo "An unrelated error occurred: ${err.getMessage()}" throw err } } sh 'sleep 1s' script { try { - // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 10, unit: 'MINUTES') { + // Allow 5 minutes for the test to run + timeout(time: 5, unit: 'MINUTES') { sh '''python3 ci-scripts/hackrf_pro_test.py --ci --log log \ --hostdir host/build/hackrf-tools/src \ --fwupdate firmware/hackrf_usb/build \ --tester 0000000000000000a06063c82338145f \ --eut RunningFromRAM -p --rev r1.2''' } - } catch (err) { - echo "HackRF Pro Test timeout reached." - throw err - } - } - - script { - try { - // Allow 10 seconds for the USB hub port power server to respond - timeout(time: 20, unit: 'SECONDS') { - sh 'hubs all off' + } catch (FlowInterruptedException err) { + // Check if the cause was specifically an exceeded timeout + def cause = err.getCauses().get(0) + if (cause instanceof org.jenkinsci.plugins.workflow.steps.TimeoutStepExecution.ExceededTimeout) { + echo "HackRF Pro Test timeout limit reached." + throw err // Re-throw the exception to fail the build + } else { + echo "Build interrupted for another reason." + throw err // Re-throw the exception to fail the build } - } catch (err) { - echo "USB hub port power server command timeout reached." + } catch (Exception err) { + echo "An unrelated error occurred: ${err.getMessage()}" throw err } }