diff --git a/buildroot/bin/build_all_examples b/buildroot/bin/build_all_examples index d50937a732..9b221f7d43 100755 --- a/buildroot/bin/build_all_examples +++ b/buildroot/bin/build_all_examples @@ -158,7 +158,9 @@ shopt -s nullglob export PAUSE=1 # Get a list of all folders that contain a file matching "Configuration*.h" -find -ds "$CBASE"/config/examples -type d -name 'Configuration.h' -o -name 'Configuration_adv.h' -print0 | while IFS= read -r -d $'\0' CONF; do +find "$CBASE"/config/examples -type d -name 'Configuration.h' -o -name 'Configuration_adv.h' -print0 \ + | LC_ALL=C sort -fz \ + | while IFS= read -r -d $'\0' CONF; do # Remove the file name and slash from the end of the path CONF=${CONF%/*} diff --git a/buildroot/bin/build_example b/buildroot/bin/build_example index 133f3c4cc9..809f3d565e 100755 --- a/buildroot/bin/build_example +++ b/buildroot/bin/build_example @@ -11,6 +11,7 @@ # [-o|--output] - Redirect export / archiving to another location # (By default export to origin config folder) # [-f|--nofail] - Don't stop on a failed build +# [-x|--noxfer] - Skip the build if the env ends with _xfer. Implied with --many. # [-w|--nowarn] - Suppress warnings with extra config options # [-r|--reveal] - Reveal the config/export folder after the build # [-h|--help] - Print usage and exit @@ -28,6 +29,7 @@ build_example -b|--base= - Configurations root folder (e.g., ./.pio/bu [-o|--output] - Redirect export / archiving to another location (By default export to origin config folder) [-f|--nofail] - Don't stop on a failed build + [-x|--noxfer] - Skip the build if the env ends with _xfer. Implied with --many. [-w|--nowarn] - Suppress warnings with extra config options [-r|--reveal] - Reveal the config/export folder after the build [-h|--help] - Print usage and exit @@ -56,8 +58,9 @@ NOFAIL= OUTBASE= BUILDINDEX=1 MANY= +NOXFER= -while getopts 'ab:c:e:fhmn:o:r-:' OFLAG; do +while getopts 'ab:c:e:fhmn:o:r-:x' OFLAG; do case "${OFLAG}" in a) ARCHIVE=1 ;; b) BASE="${OPTARG%/}" ;; @@ -65,23 +68,25 @@ while getopts 'ab:c:e:fhmn:o:r-:' OFLAG; do e) EXPNUM="$OPTARG" ;; f) NOFAIL=1 ;; h) EXIT_USAGE=1 ; break ;; - m) MANY=1 ;; + m) MANY=1 ; NOXFER=1 ;; n) BUILDINDEX="$OPTARG" ;; o) OUTBASE="${OPTARG%/}" ;; r) REVEAL=1 ;; + x) NOXFER=1 ;; -) ONAM="${OPTARG%%=*}" ; OVAL="${OPTARG#*=}" case "$ONAM" in archive) ARCHIVE=1 ;; allow) ALLOW=1 ;; base) BASE="${OVAL%/}" ;; config) CONFIG="${OVAL%/}" ;; - many) MANY=1 ;; + many) MANY=1 ; NOXFER=1 ;; index) BUILDINDEX="$OVAL" ;; export) EXPNUM="$OVAL" ;; output) OUTBASE="${OVAL%/}" ;; help) EXIT_USAGE=1 ; break ;; nofail) NOFAIL=1 ;; reveal) REVEAL=1 ;; + noxfer) NOXFER=1 ;; *) EXIT_USAGE=2 ; echo "$SELF: unrecognized option \`--$ONAM'" ; break ;; esac ;; @@ -200,83 +205,92 @@ fi # Build all from BUILDINDEX onward (usually 1) meaning ALL. # MANY with a BUILDINDEX may be useful for continuing an interrupted build. -while ((1)); do +while true; do set +e - echo "Building example $CONFIG ($BUILDINDEX)..." + # Skip over "_xfer" environments when specified + if [[ -n $NOXFER && ${ENVARRAY[BUILDINDEX-1]} =~ _xfer$ ]]; then - # Run a build and record the error number - mftest -s -a -n$BUILDINDEX ; ERR=$? - - # "Index out of range" can fail without an error - ((MANY)) && ((ERR == 66)) && ERR=0 && break # "index out of range" - - set -e - - if [[ $ERR -gt 0 ]]; then - - alrt "Failed ($ERR)" - - # Error? For --nofail simply log. Otherwise return the error. - if [[ -n $NOFAIL ]]; then - date +"%F %T [FAIL] $CONFIG ($BUILDINDEX)" >>./.pio/error-log.txt - else - exit $ERR - fi + echo "Skipping example $CONFIG ($BUILDINDEX)..." else - annc "Success" + echo "Building example $CONFIG ($BUILDINDEX)..." - # Copy exports back to the configs - if [[ -n $EXPNUM ]]; then - annc "Exporting $EXPNUM" - [[ -f Marlin/Config-export.h ]] && { cp Marlin/Config-export.h "$ARCSUB"/Config.h ; } - find "$BUILD" -type f \( "${ENAME[@]}" \) -exec cp "{}" "$ARCSUB" \; - fi + # Run a build and record the error number + mftest -s -a -n$BUILDINDEX ; ERR=$? + + # "Index out of range" can fail without an error + ((MANY)) && ((ERR == 66)) && ERR=0 && break # "index out of range" + + set -e + + if [[ $ERR -gt 0 ]]; then + + alrt "Failed ($ERR)" + + # Error? For --nofail simply log. Otherwise return the error. + if [[ -n $NOFAIL ]]; then + date +"%F %T [FAIL] $CONFIG ($BUILDINDEX)" >>./.pio/error-log.txt + else + exit $ERR + fi - # When building many, create sub-folders for each build env name - if [[ -n $MANY && $ENVCOUNT -gt 1 ]]; then - ENV=${ENVARRAY[BUILDINDEX-1]} - ARCENVSUB="$ARCSUB/$ENV" else - ARCENVSUB="$ARCSUB" - fi - # Copy potential firmware files into the config folder - # TODO: Consider firmware that needs an STM32F4_UPDATE folder. - # Currently only BOARD_CREALITY_F401RE env:STM32F401RE_creality - if ((ARCHIVE)); then - annc "Archiving" - find "$BUILD" -type f \( "${BNAME[@]}" \) -exec sh -c ' - ARCDIR="$1" ; CONFIG="$2" ; FILE="$3" ; shift 3 - NAME=${FILE##*/} ; SHRT=${NAME%.*} ; DIR=${FILE%/*} - ZIPX= - if [[ $CONFIG == *Simulator* ]]; then - case $(uname | tr '[:upper:]' '[:lower:]') in - darwin) SUB="macOS" ; ZIPX="-X" ;; - *linux) SUB="Linux" ;; - win*) SUB="Windows" ;; - msys*) SUB="Windows" ;; - cygwin*) SUB="Windows" ;; - mingw*) SUB="Windows" ;; - *) SUB='Unix' ;; - esac - ARCH=$(uname -m | tr '[:lower:]' '[:upper:]') - ARCDIR="$ARCDIR/$SUB-$ARCH" - fi - mkdir -p "$ARCDIR" - rm -f "$ARCDIR"/*.zip "$ARCDIR"/*.sha256.txt - cd "$DIR" - SHASUM=$(sha256sum "$NAME" | cut -d" " -f1) - echo "$CONFIG\n$SHASUM" > "$ARCDIR/$NAME.sha256.txt" - zip $ZIPX "$ARCDIR/$SHRT.zip" "$NAME" && rm "$NAME" - cd - >/dev/null - ' sh "$ARCENVSUB" "$CONFIG" {} + - fi + annc "Success" - # Reveal the configs after the build, if requested - ((REVEAL)) && { annc "Revealing $ARCENVSUB" ; open "$ARCENVSUB" ; } + # Copy exports back to the configs + if [[ -n $EXPNUM ]]; then + annc "Exporting $EXPNUM" + [[ -f Marlin/Config-export.h ]] && { cp Marlin/Config-export.h "$ARCSUB"/Config.h ; } + find "$BUILD" -type f \( "${ENAME[@]}" \) -exec cp "{}" "$ARCSUB" \; + fi + + # When building many, create sub-folders for each build env name + if [[ -n $MANY ]]; then + ENV=${ENVARRAY[BUILDINDEX-1]} + ARCENVSUB="$ARCSUB/$ENV" + else + ARCENVSUB="$ARCSUB" + fi + + # Copy potential firmware files into the config folder + # TODO: Consider firmware that needs an STM32F4_UPDATE folder. + # Currently only BOARD_CREALITY_F401RE env:STM32F401RE_creality + if ((ARCHIVE)); then + annc "Archiving" + find "$BUILD" -type f \( "${BNAME[@]}" \) -exec sh -c ' + ARCDIR="$1" ; CONFIG="$2" ; FILE="$3" ; shift 3 + NAME=${FILE##*/} ; SHRT=${NAME%.*} ; DIR=${FILE%/*} + ZIPX= + if [[ $CONFIG == *Simulator* ]]; then + case $(uname | tr '[:upper:]' '[:lower:]') in + darwin) SUB="macOS" ; ZIPX="-X" ;; + *linux) SUB="Linux" ;; + win*) SUB="Windows" ;; + msys*) SUB="Windows" ;; + cygwin*) SUB="Windows" ;; + mingw*) SUB="Windows" ;; + *) SUB='Unix' ;; + esac + ARCH=$(uname -m | tr '[:lower:]' '[:upper:]') + ARCDIR="$ARCDIR/$SUB-$ARCH" + fi + mkdir -p "$ARCDIR" + rm -f "$ARCDIR"/*.zip "$ARCDIR"/*.sha256.txt + cd "$DIR" + SHASUM=$(sha256sum "$NAME" | cut -d" " -f1) + echo "$CONFIG\n$SHASUM" > "$ARCDIR/$NAME.sha256.txt" + zip $ZIPX "$ARCDIR/$SHRT.zip" "$NAME" && rm "$NAME" + cd - >/dev/null + ' sh "$ARCENVSUB" "$CONFIG" {} + + fi # ARCHIVE + + # Reveal the configs after the build, if requested + ((REVEAL)) && { annc "Revealing $ARCENVSUB" ; open "$ARCENVSUB" ; } + + fi # Success fi diff --git a/buildroot/bin/mfenvs b/buildroot/bin/mfenvs index 3c726af53c..38e5d55405 100755 --- a/buildroot/bin/mfenvs +++ b/buildroot/bin/mfenvs @@ -1,6 +1,7 @@ #!/usr/bin/env bash # # mfenvs Print the current board and environment information +# Use -n to remove "*_xfer" environments from the list. # Output -> "SHORT_NAME (###): [ env1 env2 env3 ... ]" # @@ -27,6 +28,7 @@ BLINE=$( grep -E "define\s+BOARD_$MB\b" Marlin/src/core/boards.h ) BNUM=$( sed -E 's/^.+BOARD_[^ ]+ +([0-9]+).+$/\1/' <<<"$BLINE" ) [[ -z $BNUM ]] && { echo "Error - Can't find BOARD_$MB in core/boards.h." ; exit 1 ; } ENVS=( $( grep -EA1 "MB\(.*\b$MB\b.*\)" Marlin/src/pins/pins.h | grep -E "#include.+//.+(env|$SYS):[^ ]+" | grep -oE "(env|$SYS):[^ ]+" | sed -E "s/(env|$SYS)://" ) ) +[[ "$1" = "-n" ]] && ENVS=( $(printf "%s\n" "${ENVS[@]}" | grep -v "_xfer$") ) [[ -z $ENVS ]] && { errout "Error - Can't find target(s) for $MB ($BNUM)." ; exit 1 ; } ECOUNT=${#ENVS[*]} [[ $ECOUNT == 1 ]] && EOUT=$ENVS || EOUT="${ENVS[@]}" diff --git a/buildroot/bin/use_example_configs b/buildroot/bin/use_example_configs index 282e057dde..72930d8c45 100755 --- a/buildroot/bin/use_example_configs +++ b/buildroot/bin/use_example_configs @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/env python # # use_example_configs [repo:]configpath # @@ -14,40 +14,127 @@ # The fallback branch is bugfix-2.1.x. # -which curl >/dev/null && TOOL='curl -L -s -S -f -o wgot' -which wget >/dev/null && TOOL='wget -q -O wgot' +import os, subprocess, sys, urllib.request, urllib.error +from pathlib import Path -CURR=$(git branch 2>/dev/null | grep ^* | sed 's/\* //g') -case "$CURR" in - bugfix-2.*.x ) BRANCH=$CURR ;; - *-2.1.x|2.1.x ) BRANCH=latest-2.1.x ;; - *-2.0.x|2.0.x ) BRANCH=latest-2.0.x ;; - *-1.1.x|1.1.x ) BRANCH=latest-1.1.x ;; - *-1.0.x|1.0.x ) BRANCH=latest-1.0.x ;; - * ) BRANCH=bugfix-2.1.x ;; -esac +DEBUGGING = False +CONFIG_FILES = ("Configuration.h", "Configuration_adv.h", "_Bootscreen.h", "_Statusscreen.h") -if [[ $# > 0 ]]; then - IFS=: read -r PART1 PART2 <<< "$@" - [[ -n $PART2 ]] && { UDIR="$PART2" ; BRANCH="$PART1" ; } \ - || { UDIR="$PART1" ; } - RDIR="${UDIR// /%20}" - echo "Fetching $UDIR configurations from $BRANCH..." - EXAMPLES="examples/$RDIR" -else - EXAMPLES="default" -fi +def debug_print(s): + if DEBUGGING: print(s) -CONFIGS="https://raw.githubusercontent.com/MarlinFirmware/Configurations/$BRANCH/config/${EXAMPLES}" +def get_current_branch(): + try: + result = subprocess.run(['git', 'branch'], capture_output=True, text=True, check=True) + for line in result.stdout.splitlines(): + if line.startswith('*'): + return line[2:] + except subprocess.CalledProcessError: + return None -restore_configs +def sparse_checkout(branch, config_path, repo_url="https://github.com/MarlinFirmware/Configurations.git"): + configs_dir = Path("ConfigurationsRepo") + config_subdir = f"config/{config_path}" -cd Marlin + if not configs_dir.exists(): + # Step 1: Clone with no checkout + subprocess.run([ + "git", "clone", "--depth", "1", "--filter=blob:none", "--sparse", + "--branch", branch, repo_url, str(configs_dir) + ], check=True) -$TOOL "$CONFIGS/Configuration.h" >/dev/null 2>&1 && mv wgot Configuration.h -$TOOL "$CONFIGS/Configuration_adv.h" >/dev/null 2>&1 && mv wgot Configuration_adv.h -$TOOL "$CONFIGS/_Bootscreen.h" >/dev/null 2>&1 && mv wgot _Bootscreen.h -$TOOL "$CONFIGS/_Statusscreen.h" >/dev/null 2>&1 && mv wgot _Statusscreen.h + # Step 2: Enable sparse checkout and set the folder + subprocess.run(["git", "sparse-checkout", "set", config_subdir], cwd=str(configs_dir), check=True) + # Step 3: Pull the latest for that branch/folder + subprocess.run(["git", "pull"], cwd=str(configs_dir), check=True) -rm -f wgot -cd - >/dev/null +def copy_config_files(branch, config_path, dest_dir): + sparse_checkout(branch, config_path) + + src_dir = Path("ConfigurationsRepo") / "config" / config_path + for fname in CONFIG_FILES: + src_file = src_dir / fname + if src_file.exists(): + dest_file = dest_dir / fname + debug_print(f"Copying {src_file} to {dest_file}") + dest_file.write_bytes(src_file.read_bytes()) + else: + debug_print(f"{fname} not found in {src_dir}") + +def fetch_config_files(branch, config_path, dest_dir): + config_path_url = config_path.replace(' ', '%20') + base_url = f"https://raw.githubusercontent.com/MarlinFirmware/Configurations/{branch}/config/{config_path_url}" + + for file in CONFIG_FILES: + url = f"{base_url}/{file}" + dest_file = dest_dir / file + if os.getenv('DEBUG', '0') == '1': + debug_print(f"Fetching {file} from {url} to {dest_file}") + + try: + urllib.request.urlretrieve(url, dest_file) + except urllib.error.HTTPError as e: + if e.code == 404: + if os.getenv('DEBUG', '0') == '1': + print(f"File {file} not found (404), skipping.") + else: + raise + +def fetch_configs(branch, config_path): + print(f"Fetching {config_path} configurations from {branch}...") + + marlin_dir = Path("Marlin") + if not marlin_dir.exists(): + print(f"Directory 'Marlin' not found at the current location.") + sys.exit(1) + + if os.environ.get('GITHUB_ACTIONS'): # Running on GitHub ? + copy_config_files(branch, config_path, marlin_dir) + else: + fetch_config_files(branch, config_path, marlin_dir) + +def main(): + branch = get_current_branch() + if not branch: + print("Not a git repository or no branch found.") + sys.exit(1) + + if branch.startswith("bugfix-2."): + branch = branch + elif branch.endswith("bugfix-2.1.x"): + branch = "bugfix-2.1.x" + elif branch.endswith("-2.1.x") or branch == "2.1.x": + branch = "latest-2.1.x" + elif branch.endswith("-2.0.x") or branch == "2.0.x": + branch = "latest-2.0.x" + elif branch.endswith("-1.1.x") or branch == "1.1.x": + branch = "latest-1.1.x" + elif branch.endswith("-1.0.x") or branch == "1.0.x": + branch = "latest-1.0.x" + else: + branch = "bugfix-2.1.x" + + if len(sys.argv) > 1: + arg = sys.argv[1] + if ':' in arg: + part1, part2 = arg.split(':', 1) + config_path = part2 + branch = part1 + else: + config_path = arg + config_path = 'examples/'+config_path + else: + config_path = "default" + + try: + subprocess.run(['restore_configs'], check=True) + except FileNotFoundError: + try: + subprocess.run(['./buildroot/bin/restore_configs'], check=True) + except FileNotFoundError: + print("restore_configs not found, skipping.") + + fetch_configs(branch, config_path) + +if __name__ == "__main__": + main()