Update build scripts

2026-02-20 00:31:35 +01:00 · 2025-12-23 19:12:37 -06:00
parent 324ce04698
commit 14708a3067
4 changed files with 206 additions and 101 deletions
--- a/buildroot/bin/build_all_examples
+++ b/buildroot/bin/build_all_examples
@@ -158,7 +158,9 @@ shopt -s nullglob
 export PAUSE=1

 # Get a list of all folders that contain a file matching "Configuration*.h"
-find -ds "$CBASE"/config/examples -type d -name 'Configuration.h' -o -name 'Configuration_adv.h' -print0 | while IFS= read -r -d $'\0' CONF; do
+find "$CBASE"/config/examples -type d -name 'Configuration.h' -o -name 'Configuration_adv.h' -print0 \
+  | LC_ALL=C sort -fz \
+  | while IFS= read -r -d $'\0' CONF; do

  # Remove the file name and slash from the end of the path
  CONF=${CONF%/*}
--- a/buildroot/bin/build_example
+++ b/buildroot/bin/build_example
@@ -11,6 +11,7 @@
 #                [-o|--output]        - Redirect export / archiving to another location
 #                                       (By default export to origin config folder)
 #                [-f|--nofail]        - Don't stop on a failed build
+#                [-x|--noxfer]        - Skip the build if the env ends with _xfer. Implied with --many.
 #                [-w|--nowarn]        - Suppress warnings with extra config options
 #                [-r|--reveal]        - Reveal the config/export folder after the build
 #                [-h|--help]          - Print usage and exit
@@ -28,6 +29,7 @@ build_example  -b|--base=<path>    - Configurations root folder (e.g., ./.pio/bu
              [-o|--output]        - Redirect export / archiving to another location
                                     (By default export to origin config folder)
              [-f|--nofail]        - Don't stop on a failed build
+              [-x|--noxfer]        - Skip the build if the env ends with _xfer. Implied with --many.
              [-w|--nowarn]        - Suppress warnings with extra config options
              [-r|--reveal]        - Reveal the config/export folder after the build
              [-h|--help]          - Print usage and exit
@@ -56,8 +58,9 @@ NOFAIL=
 OUTBASE=
 BUILDINDEX=1
 MANY=
+NOXFER=

-while getopts 'ab:c:e:fhmn:o:r-:' OFLAG; do
+while getopts 'ab:c:e:fhmn:o:r-:x' OFLAG; do
  case "${OFLAG}" in
    a) ARCHIVE=1 ;;
    b) BASE="${OPTARG%/}" ;;
@@ -65,23 +68,25 @@ while getopts 'ab:c:e:fhmn:o:r-:' OFLAG; do
    e) EXPNUM="$OPTARG" ;;
    f) NOFAIL=1 ;;
    h) EXIT_USAGE=1 ; break ;;
-    m) MANY=1 ;;
+    m) MANY=1 ; NOXFER=1 ;;
    n) BUILDINDEX="$OPTARG" ;;
    o) OUTBASE="${OPTARG%/}" ;;
    r) REVEAL=1 ;;
+    x) NOXFER=1 ;;
    -) ONAM="${OPTARG%%=*}" ; OVAL="${OPTARG#*=}"
       case "$ONAM" in
        archive) ARCHIVE=1 ;;
          allow) ALLOW=1 ;;
           base) BASE="${OVAL%/}" ;;
         config) CONFIG="${OVAL%/}" ;;
-           many) MANY=1 ;;
+           many) MANY=1 ; NOXFER=1 ;;
          index) BUILDINDEX="$OVAL" ;;
         export) EXPNUM="$OVAL" ;;
         output) OUTBASE="${OVAL%/}" ;;
           help) EXIT_USAGE=1 ; break ;;
         nofail) NOFAIL=1 ;;
         reveal) REVEAL=1 ;;
+         noxfer) NOXFER=1 ;;
              *) EXIT_USAGE=2 ; echo "$SELF: unrecognized option \`--$ONAM'" ; break ;;
       esac
       ;;
@@ -200,83 +205,92 @@ fi
 # Build all from BUILDINDEX onward (usually 1) meaning ALL.
 # MANY with a BUILDINDEX may be useful for continuing an interrupted build.

-while ((1)); do
+while true; do
  set +e

-  echo "Building example $CONFIG ($BUILDINDEX)..."
+  # Skip over "_xfer" environments when specified
+  if [[ -n $NOXFER && ${ENVARRAY[BUILDINDEX-1]} =~ _xfer$ ]]; then

-  # Run a build and record the error number
-  mftest -s -a -n$BUILDINDEX ; ERR=$?
-
-  # "Index out of range" can fail without an error
-  ((MANY)) && ((ERR == 66)) && ERR=0 && break  # "index out of range"
-
-  set -e
-
-  if [[ $ERR -gt 0 ]]; then
-
-    alrt "Failed ($ERR)"
-
-    # Error? For --nofail simply log. Otherwise return the error.
-    if [[ -n $NOFAIL ]]; then
-      date +"%F %T [FAIL] $CONFIG ($BUILDINDEX)" >>./.pio/error-log.txt
-    else
-      exit $ERR
-    fi
+    echo "Skipping example $CONFIG ($BUILDINDEX)..."

  else

-    annc "Success"
+    echo "Building example $CONFIG ($BUILDINDEX)..."

-    # Copy exports back to the configs
-    if [[ -n $EXPNUM ]]; then
-      annc "Exporting $EXPNUM"
-      [[ -f Marlin/Config-export.h ]] && { cp Marlin/Config-export.h "$ARCSUB"/Config.h ; }
-      find "$BUILD" -type f \( "${ENAME[@]}" \) -exec cp "{}" "$ARCSUB" \;
-    fi
+    # Run a build and record the error number
+    mftest -s -a -n$BUILDINDEX ; ERR=$?
+
+    # "Index out of range" can fail without an error
+    ((MANY)) && ((ERR == 66)) && ERR=0 && break  # "index out of range"
+
+    set -e
+
+    if [[ $ERR -gt 0 ]]; then
+
+      alrt "Failed ($ERR)"
+
+      # Error? For --nofail simply log. Otherwise return the error.
+      if [[ -n $NOFAIL ]]; then
+        date +"%F %T [FAIL] $CONFIG ($BUILDINDEX)" >>./.pio/error-log.txt
+      else
+        exit $ERR
+      fi

-    # When building many, create sub-folders for each build env name
-    if [[ -n $MANY && $ENVCOUNT -gt 1 ]]; then
-      ENV=${ENVARRAY[BUILDINDEX-1]}
-      ARCENVSUB="$ARCSUB/$ENV"
    else
-      ARCENVSUB="$ARCSUB"
-    fi

-    # Copy potential firmware files into the config folder
-    # TODO: Consider firmware that needs an STM32F4_UPDATE folder.
-    #       Currently only BOARD_CREALITY_F401RE env:STM32F401RE_creality
-    if ((ARCHIVE)); then
-      annc "Archiving"
-      find "$BUILD" -type f \( "${BNAME[@]}" \) -exec sh -c '
-        ARCDIR="$1" ; CONFIG="$2" ; FILE="$3" ; shift 3
-        NAME=${FILE##*/} ; SHRT=${NAME%.*} ; DIR=${FILE%/*}
-        ZIPX=
-        if [[ $CONFIG == *Simulator* ]]; then
-          case $(uname | tr '[:upper:]' '[:lower:]') in
-            darwin) SUB="macOS" ; ZIPX="-X" ;;
-            *linux) SUB="Linux" ;;
-              win*) SUB="Windows" ;;
-             msys*) SUB="Windows" ;;
-           cygwin*) SUB="Windows" ;;
-            mingw*) SUB="Windows" ;;
-                 *) SUB='Unix' ;;
-          esac
-          ARCH=$(uname -m | tr '[:lower:]' '[:upper:]')
-          ARCDIR="$ARCDIR/$SUB-$ARCH"
-        fi
-        mkdir -p "$ARCDIR"
-        rm -f "$ARCDIR"/*.zip "$ARCDIR"/*.sha256.txt
-        cd "$DIR"
-        SHASUM=$(sha256sum "$NAME" | cut -d" " -f1)
-        echo "$CONFIG\n$SHASUM" > "$ARCDIR/$NAME.sha256.txt"
-        zip $ZIPX "$ARCDIR/$SHRT.zip" "$NAME" && rm "$NAME"
-        cd - >/dev/null
-      ' sh "$ARCENVSUB" "$CONFIG" {} +
-    fi
+      annc "Success"

-    # Reveal the configs after the build, if requested
-    ((REVEAL)) && { annc "Revealing $ARCENVSUB" ; open "$ARCENVSUB" ; }
+      # Copy exports back to the configs
+      if [[ -n $EXPNUM ]]; then
+        annc "Exporting $EXPNUM"
+        [[ -f Marlin/Config-export.h ]] && { cp Marlin/Config-export.h "$ARCSUB"/Config.h ; }
+        find "$BUILD" -type f \( "${ENAME[@]}" \) -exec cp "{}" "$ARCSUB" \;
+      fi
+
+      # When building many, create sub-folders for each build env name
+      if [[ -n $MANY ]]; then
+        ENV=${ENVARRAY[BUILDINDEX-1]}
+        ARCENVSUB="$ARCSUB/$ENV"
+      else
+        ARCENVSUB="$ARCSUB"
+      fi
+
+      # Copy potential firmware files into the config folder
+      # TODO: Consider firmware that needs an STM32F4_UPDATE folder.
+      #       Currently only BOARD_CREALITY_F401RE env:STM32F401RE_creality
+      if ((ARCHIVE)); then
+        annc "Archiving"
+        find "$BUILD" -type f \( "${BNAME[@]}" \) -exec sh -c '
+          ARCDIR="$1" ; CONFIG="$2" ; FILE="$3" ; shift 3
+          NAME=${FILE##*/} ; SHRT=${NAME%.*} ; DIR=${FILE%/*}
+          ZIPX=
+          if [[ $CONFIG == *Simulator* ]]; then
+            case $(uname | tr '[:upper:]' '[:lower:]') in
+              darwin) SUB="macOS" ; ZIPX="-X" ;;
+              *linux) SUB="Linux" ;;
+                win*) SUB="Windows" ;;
+               msys*) SUB="Windows" ;;
+             cygwin*) SUB="Windows" ;;
+              mingw*) SUB="Windows" ;;
+                   *) SUB='Unix' ;;
+            esac
+            ARCH=$(uname -m | tr '[:lower:]' '[:upper:]')
+            ARCDIR="$ARCDIR/$SUB-$ARCH"
+          fi
+          mkdir -p "$ARCDIR"
+          rm -f "$ARCDIR"/*.zip "$ARCDIR"/*.sha256.txt
+          cd "$DIR"
+          SHASUM=$(sha256sum "$NAME" | cut -d" " -f1)
+          echo "$CONFIG\n$SHASUM" > "$ARCDIR/$NAME.sha256.txt"
+          zip $ZIPX "$ARCDIR/$SHRT.zip" "$NAME" && rm "$NAME"
+          cd - >/dev/null
+        ' sh "$ARCENVSUB" "$CONFIG" {} +
+      fi # ARCHIVE
+
+      # Reveal the configs after the build, if requested
+      ((REVEAL)) && { annc "Revealing $ARCENVSUB" ; open "$ARCENVSUB" ; }
+
+    fi # Success

  fi

--- a/buildroot/bin/mfenvs
+++ b/buildroot/bin/mfenvs
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 #
 #  mfenvs   Print the current board and environment information
+#           Use -n to remove "*_xfer" environments from the list.
 #           Output -> "SHORT_NAME (###): [ env1 env2 env3 ... ]"
 #

@@ -27,6 +28,7 @@ BLINE=$( grep -E "define\s+BOARD_$MB\b" Marlin/src/core/boards.h )
 BNUM=$( sed -E 's/^.+BOARD_[^ ]+ +([0-9]+).+$/\1/' <<<"$BLINE" )
 [[ -z $BNUM ]] && { echo "Error - Can't find BOARD_$MB in core/boards.h." ; exit 1 ; }
 ENVS=( $( grep -EA1 "MB\(.*\b$MB\b.*\)" Marlin/src/pins/pins.h | grep -E "#include.+//.+(env|$SYS):[^ ]+" | grep -oE "(env|$SYS):[^ ]+" | sed -E "s/(env|$SYS)://" ) )
+[[ "$1" = "-n" ]] && ENVS=( $(printf "%s\n" "${ENVS[@]}" | grep -v "_xfer$") )
 [[ -z $ENVS ]] && { errout "Error - Can't find target(s) for $MB ($BNUM)." ; exit 1 ; }
 ECOUNT=${#ENVS[*]}
 [[ $ECOUNT == 1 ]] && EOUT=$ENVS || EOUT="${ENVS[@]}"
--- a/buildroot/bin/use_example_configs
+++ b/buildroot/bin/use_example_configs
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env python
 #
 # use_example_configs [repo:]configpath
 #
@@ -14,40 +14,127 @@
 #       The fallback branch is bugfix-2.1.x.
 #

-which curl >/dev/null && TOOL='curl -L -s -S -f -o wgot'
-which wget >/dev/null && TOOL='wget -q -O wgot'
+import os, subprocess, sys, urllib.request, urllib.error
+from pathlib import Path

-CURR=$(git branch 2>/dev/null | grep ^* | sed 's/\* //g')
-case "$CURR" in
-   bugfix-2.*.x ) BRANCH=$CURR ;;
-  *-2.1.x|2.1.x ) BRANCH=latest-2.1.x ;;
-  *-2.0.x|2.0.x ) BRANCH=latest-2.0.x ;;
-  *-1.1.x|1.1.x ) BRANCH=latest-1.1.x ;;
-  *-1.0.x|1.0.x ) BRANCH=latest-1.0.x ;;
-              * ) BRANCH=bugfix-2.1.x ;;
-esac
+DEBUGGING = False
+CONFIG_FILES = ("Configuration.h", "Configuration_adv.h", "_Bootscreen.h", "_Statusscreen.h")

-if [[ $# > 0 ]]; then
-  IFS=: read -r PART1 PART2 <<< "$@"
-  [[ -n $PART2 ]] && { UDIR="$PART2" ; BRANCH="$PART1" ; } \
-                  || { UDIR="$PART1" ; }
-  RDIR="${UDIR// /%20}"
-  echo "Fetching $UDIR configurations from $BRANCH..."
-  EXAMPLES="examples/$RDIR"
-else
-  EXAMPLES="default"
-fi
+def debug_print(s):
+    if DEBUGGING: print(s)

-CONFIGS="https://raw.githubusercontent.com/MarlinFirmware/Configurations/$BRANCH/config/${EXAMPLES}"
+def get_current_branch():
+    try:
+        result = subprocess.run(['git', 'branch'], capture_output=True, text=True, check=True)
+        for line in result.stdout.splitlines():
+            if line.startswith('*'):
+                return line[2:]
+    except subprocess.CalledProcessError:
+        return None

-restore_configs
+def sparse_checkout(branch, config_path, repo_url="https://github.com/MarlinFirmware/Configurations.git"):
+    configs_dir = Path("ConfigurationsRepo")
+    config_subdir = f"config/{config_path}"

-cd Marlin
+    if not configs_dir.exists():
+        # Step 1: Clone with no checkout
+        subprocess.run([
+            "git", "clone", "--depth", "1", "--filter=blob:none", "--sparse",
+            "--branch", branch, repo_url, str(configs_dir)
+        ], check=True)

-$TOOL "$CONFIGS/Configuration.h"     >/dev/null 2>&1 && mv wgot Configuration.h
-$TOOL "$CONFIGS/Configuration_adv.h" >/dev/null 2>&1 && mv wgot Configuration_adv.h
-$TOOL "$CONFIGS/_Bootscreen.h"       >/dev/null 2>&1 && mv wgot _Bootscreen.h
-$TOOL "$CONFIGS/_Statusscreen.h"     >/dev/null 2>&1 && mv wgot _Statusscreen.h
+    # Step 2: Enable sparse checkout and set the folder
+    subprocess.run(["git", "sparse-checkout", "set", config_subdir], cwd=str(configs_dir), check=True)
+    # Step 3: Pull the latest for that branch/folder
+    subprocess.run(["git", "pull"], cwd=str(configs_dir), check=True)

-rm -f wgot
-cd - >/dev/null
+def copy_config_files(branch, config_path, dest_dir):
+    sparse_checkout(branch, config_path)
+
+    src_dir = Path("ConfigurationsRepo") / "config" / config_path
+    for fname in CONFIG_FILES:
+        src_file = src_dir / fname
+        if src_file.exists():
+            dest_file = dest_dir / fname
+            debug_print(f"Copying {src_file} to {dest_file}")
+            dest_file.write_bytes(src_file.read_bytes())
+        else:
+            debug_print(f"{fname} not found in {src_dir}")
+
+def fetch_config_files(branch, config_path, dest_dir):
+    config_path_url = config_path.replace(' ', '%20')
+    base_url = f"https://raw.githubusercontent.com/MarlinFirmware/Configurations/{branch}/config/{config_path_url}"
+
+    for file in CONFIG_FILES:
+        url = f"{base_url}/{file}"
+        dest_file = dest_dir / file
+        if os.getenv('DEBUG', '0') == '1':
+            debug_print(f"Fetching {file} from {url} to {dest_file}")
+
+        try:
+            urllib.request.urlretrieve(url, dest_file)
+        except urllib.error.HTTPError as e:
+            if e.code == 404:
+                if os.getenv('DEBUG', '0') == '1':
+                    print(f"File {file} not found (404), skipping.")
+            else:
+                raise
+
+def fetch_configs(branch, config_path):
+    print(f"Fetching {config_path} configurations from {branch}...")
+
+    marlin_dir = Path("Marlin")
+    if not marlin_dir.exists():
+        print(f"Directory 'Marlin' not found at the current location.")
+        sys.exit(1)
+
+    if os.environ.get('GITHUB_ACTIONS'):        # Running on GitHub ?
+        copy_config_files(branch, config_path, marlin_dir)
+    else:
+        fetch_config_files(branch, config_path, marlin_dir)
+
+def main():
+    branch = get_current_branch()
+    if not branch:
+        print("Not a git repository or no branch found.")
+        sys.exit(1)
+
+    if branch.startswith("bugfix-2."):
+        branch = branch
+    elif branch.endswith("bugfix-2.1.x"):
+        branch = "bugfix-2.1.x"
+    elif branch.endswith("-2.1.x") or branch == "2.1.x":
+        branch = "latest-2.1.x"
+    elif branch.endswith("-2.0.x") or branch == "2.0.x":
+        branch = "latest-2.0.x"
+    elif branch.endswith("-1.1.x") or branch == "1.1.x":
+        branch = "latest-1.1.x"
+    elif branch.endswith("-1.0.x") or branch == "1.0.x":
+        branch = "latest-1.0.x"
+    else:
+        branch = "bugfix-2.1.x"
+
+    if len(sys.argv) > 1:
+        arg = sys.argv[1]
+        if ':' in arg:
+            part1, part2 = arg.split(':', 1)
+            config_path = part2
+            branch = part1
+        else:
+            config_path = arg
+        config_path = 'examples/'+config_path
+    else:
+        config_path = "default"
+
+    try:
+        subprocess.run(['restore_configs'], check=True)
+    except FileNotFoundError:
+        try:
+            subprocess.run(['./buildroot/bin/restore_configs'], check=True)
+        except FileNotFoundError:
+            print("restore_configs not found, skipping.")
+
+    fetch_configs(branch, config_path)
+
+if __name__ == "__main__":
+    main()