diff --git a/code/espurna/config/version.h b/code/espurna/config/version.h index 1af82b3b..e06e36c6 100644 --- a/code/espurna/config/version.h +++ b/code/espurna/config/version.h @@ -4,8 +4,22 @@ #pragma once +#ifndef APP_NAME #define APP_NAME "ESPURNA" +#endif + +#ifndef APP_VERSION #define APP_VERSION "1.15.0-dev" +#endif + +#ifndef APP_AUTHOR #define APP_AUTHOR "xose.perez@gmail.com" +#endif + +#ifndef APP_WEBSITE #define APP_WEBSITE "http://tinkerman.cat" +#endif + +#ifndef CFG_VERSION #define CFG_VERSION 6 +#endif diff --git a/code/espurna/utils.cpp b/code/espurna/utils.cpp index aac72dde..969d82c9 100644 --- a/code/espurna/utils.cpp +++ b/code/espurna/utils.cpp @@ -79,14 +79,7 @@ const String& getCoreRevision() { } const char* getVersion() { - static const char version[] { -#if defined(APP_REVISION) - APP_VERSION APP_REVISION -#else - APP_VERSION -#endif - }; - + static const char version[] = APP_VERSION; return version; } diff --git a/code/scripts/espurna_utils/__init__.py b/code/scripts/espurna_utils/__init__.py index 9b128d62..20080afd 100644 --- a/code/scripts/espurna_utils/__init__.py +++ b/code/scripts/espurna_utils/__init__.py @@ -4,7 +4,7 @@ # Copyright (C) 2016-2019 by Xose Pérez # # ldscripts, lwip patching, updated postmortem flags and git support -# Copyright (C) 2019 by Maxim Prokhorov +# Copyright (C) 2019-2021 by Maxim Prokhorov # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,21 +19,24 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +from .build import firmware_destination, app_add_target_build_and_copy from .checks import check_cppcheck, check_printsize +from .flags import app_inject_flags from .float_support import remove_float_support from .ldscripts import ldscripts_inject_libpath from .postmortem import dummy_ets_printf -from .git import app_inject_revision -from .release import copy_release -from .flags import app_inject_flags +from .version import app_inject_version, app_full_version_for_env __all__ = [ + "app_add_target_build_and_copy", + "app_full_version_for_env", + "app_inject_flags", + "app_inject_version", + "app_version", "check_cppcheck", "check_printsize", - "remove_float_support", - "ldscripts_inject_libpath", "dummy_ets_printf", - "app_inject_revision", - "app_inject_flags", - "copy_release", + "firmware_destination", + "ldscripts_inject_libpath", + "remove_float_support", ] diff --git a/code/scripts/espurna_utils/build.py b/code/scripts/espurna_utils/build.py new file mode 100644 index 00000000..301da0b4 --- /dev/null +++ b/code/scripts/espurna_utils/build.py @@ -0,0 +1,64 @@ +import atexit +import os +import shutil +import tempfile +import functools + +from .display import print_warning +from .version import app_full_version_for_env + + +def try_remove(path): + try: + os.remove(path) + except: # pylint: disable=bare-except + print_warning("Please manually remove the file `{}`".format(path)) + + +# emulate .ino concatenation to speed up compilation times +def merge_cpp(sources, output): + with tempfile.TemporaryFile() as tmp: + tmp.write(b"// !!! Automatically generated file; DO NOT EDIT !!! \n") + tmp.write(b'#include "espurna.h"\n') + for source in sources: + src_include = '#include "{}"\n'.format(source) + tmp.write(src_include.encode("utf-8")) + + tmp.seek(0) + + with open(output, "wb") as fobj: + shutil.copyfileobj(tmp, fobj) + atexit.register(try_remove, output) + + +def firmware_prefix(env): + return "espurna-{}".format(app_full_version_for_env(env)) + + +# generate an common name for the current build +def firmware_filename(env): + suffix = "{}.bin".format(env["ESPURNA_BUILD_NAME"] or env["PIOENV"]) + return "-".join([firmware_prefix(env), suffix]) + + +def firmware_destination(env): + destdir = env["ESPURNA_BUILD_DESTINATION"] or env["PROJECT_DIR"] + subdir = os.path.join(destdir, firmware_prefix(env)) + + return os.path.join(subdir, firmware_filename(env)) + + +def app_add_target_build_and_copy(env): + from SCons.Script import Copy + + copy_dest = firmware_destination(env) + copy = env.Command( + copy_dest, "${BUILD_DIR}/${PROGNAME}.bin", Copy("$TARGET", "$SOURCE") + ) + env.AddTarget( + "build-and-copy", + copy_dest, + actions=None, # command invocation already handles this + title="Build firmware.bin and store a copy", + description="Build and store firmware.bin as $ESPURNA_BUILD_DESTINATION/espurna--$ESPURNA_BUILD_NAME.bin (default destination is $PROJECT_DIR)", + ) diff --git a/code/scripts/espurna_utils/git.py b/code/scripts/espurna_utils/git.py deleted file mode 100644 index e4d9824a..00000000 --- a/code/scripts/espurna_utils/git.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import subprocess - -def git(*args): - cmd = ["git"] - cmd.extend(args) - proc = subprocess.Popen( - cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True - ) - return proc.stdout.readlines()[0].strip() - -def app_inject_revision(env): - revision = env.get("ESPURNA_RELEASE_REVISION", "") - if not revision: - try: - revision = ".git" + git("rev-parse", "--short=8", "HEAD") - except: # pylint: disable=broad-except - pass - - # Note: code expects this as undefined when empty - if revision: - env.Append(CPPDEFINES=[ - ("APP_REVISION", "\\\"{}\\\"".format(revision)) - ]) diff --git a/code/scripts/espurna_utils/release.py b/code/scripts/espurna_utils/release.py deleted file mode 100644 index 44b7c16c..00000000 --- a/code/scripts/espurna_utils/release.py +++ /dev/null @@ -1,49 +0,0 @@ -import atexit -import os -import shutil -import tempfile - -from .display import print_warning - - -def try_remove(path): - try: - os.remove(path) - except: # pylint: disable=bare-except - print_warning("Please manually remove the file `{}`".format(path)) - - -def copy_release(target, source, env): - # target filename and subdir for release files - name = env["ESPURNA_RELEASE_NAME"] - version = env["ESPURNA_RELEASE_VERSION"] - destdir = env["ESPURNA_RELEASE_DESTINATION"] - - if not name or not version or not destdir: - raise ValueError("Cannot set up release without release variables present") - - if not os.path.exists(destdir): - os.makedirs(destdir) - - dest = os.path.join( - destdir, "espurna-{version}-{name}.bin".format(version=version, name=name) - ) - src = env.subst("$BUILD_DIR/${PROGNAME}.bin") - - shutil.copy(src, dest) - - -# emulate .ino concatenation to speed up compilation times -def merge_cpp(sources, output): - with tempfile.TemporaryFile() as tmp: - tmp.write(b"// !!! Automatically generated file; DO NOT EDIT !!! \n") - tmp.write(b'#include "espurna.h"\n') - for source in sources: - src_include = '#include "{}"\n'.format(source) - tmp.write(src_include.encode('utf-8')) - - tmp.seek(0) - - with open(output, "wb") as fobj: - shutil.copyfileobj(tmp, fobj) - atexit.register(try_remove, output) diff --git a/code/scripts/espurna_utils/version.py b/code/scripts/espurna_utils/version.py new file mode 100644 index 00000000..ba3dda1d --- /dev/null +++ b/code/scripts/espurna_utils/version.py @@ -0,0 +1,89 @@ +import os +import functools +import subprocess + +from .display import print_warning + + +try: + cached = functools.cache +except AttributeError: + cached = functools.lru_cache(None) + + +@cached +def app_revision(): + def git(*args): + cmd = ["git"] + cmd.extend(args) + proc = subprocess.Popen( + cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True + ) + return proc.stdout.readlines()[0].strip() + + revision = None + try: + revision = git("rev-parse", "--short=8", "HEAD") + except subprocess.CalledProcessError: + pass + except FileNotFoundError: + pass + + return revision + + +@cached +def app_version(version_h): + version = None + with open(version_h, "r") as f: + for line in f: + if "define" in line and "APP_VERSION" in line: + version = line.split(" ")[-1] + version = version.strip().replace('"', "") + break + + return version + + +def app_version_for_env(env): + return env.get("ESPURNA_BUILD_VERSION") or app_version( + os.path.join(env.get("PROJECT_DIR"), "espurna/config/version.h") + ) + + +def app_revision_for_env(env): + return env.get("ESPURNA_BUILD_REVISION") or app_revision() + + +def app_suffix_for_env(env): + return env.get("ESPURNA_BUILD_VERSION_SUFFIX", "") + + +def app_combined_version(env): + version = app_version_for_env(env) + if not version: + raise ValueError("Version string cannot be empty") + + revision = app_revision_for_env(env) + if revision: + # handle both 1.2.3-dev.git... and 1.2.3-git... + # and avoid 1.2.3.git... that cannot be parsed by the semantic_version module + middle = ".git" if "-" in version else "-git" + version = middle.join([version, revision]) + + suffix = app_suffix_for_env(env) + if suffix: + version = "+".join([version, suffix]) + + return version + + +def app_full_version_for_env(env): + return env.get("ESPURNA_BUILD_FULL_VERSION") or app_combined_version(env) + + +def app_inject_version(env): + def inject_string(env, flag, value): + env.Append(CPPDEFINES=[(flag, '\\"{}\\"'.format(value))]) + + inject_string(env, "APP_VERSION", app_full_version_for_env(env)) diff --git a/code/scripts/generate_release_sh.py b/code/scripts/generate_release_sh.py index afd31d6b..d0f8c9af 100755 --- a/code/scripts/generate_release_sh.py +++ b/code/scripts/generate_release_sh.py @@ -15,14 +15,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os import argparse import re import shlex import configparser import collections -CI = "true" == os.environ.get("CI", "false") Build = collections.namedtuple("Build", "env extends build_flags src_build_flags") @@ -96,12 +94,11 @@ def generate_lines(builds, ignore): flags.append('PLATFORMIO_BUILD_FLAGS="{}"'.format(build.build_flags)) if build.src_build_flags: flags.append('ESPURNA_FLAGS="{}"'.format(build.src_build_flags)) - flags.append('ESPURNA_RELEASE_NAME="{env}"'.format(env=build.env)) - flags.append("ESPURNA_BUILD_SINGLE_SOURCE=1") + flags.append('ESPURNA_BUILD_NAME="{env}"'.format(env=build.env)) cmd = ["env"] cmd.extend(flags) - cmd.extend(["pio", "run", "-e", build.extends, "-s", "-t", "release"]) + cmd.extend(["pio", "run", "-e", build.extends, "-s", "-t", "build-and-copy"]) line = " ".join(cmd) @@ -123,22 +120,50 @@ def every(seq, nth, total): index = (index + 1) % total -if __name__ == "__main__": - if not CI: - raise ValueError("* Not in CI *") - +def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument("--version", required=True) - parser.add_argument("--destination", required=True) - parser.add_argument("--ignore", action="append") - args = parser.parse_args() + parser.add_argument( + "--destination", help="Where to place the resulting .bin", required=True + ) + parser.add_argument( + "--single-source", + help="Combine .cpp files into one to speed up compilation", + default=True, + ) + parser.add_argument( + "--ignore", help="Do not build envs that contain the string(s)", action="append" + ) + + builder_thread = parser.add_argument_group( + title="Builder thread control for CI parallel builds" + ) + builder_thread.add_argument("--builder-thread", type=int, required=True) + builder_thread.add_argument("--builder-total-threads", type=int, required=True) + + full_version = parser.add_argument_group( + title="Fully replace the version string for the build system" + ) + full_version.add_argument("--full-version") + + version_parts = parser.add_argument_group( + "Replace parts of the version string that would have been detected by the build system" + ) + version_parts.add_argument("--version") + version_parts.add_argument("--revision") + version_parts.add_argument("--suffix") + + parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() Config = configparser.ConfigParser() with open("platformio.ini", "r") as f: Config.read_file(f) - builder_total_threads = int(os.environ["BUILDER_TOTAL_THREADS"]) - builder_thread = int(os.environ["BUILDER_THREAD"]) + builder_total_threads = args.builder_total_threads + builder_thread = args.builder_thread if builder_thread >= builder_total_threads: raise ValueError("* Builder thread index out of range *") @@ -146,9 +171,18 @@ if __name__ == "__main__": print("#!/bin/bash") print("set -e -x") - print('export ESPURNA_RELEASE_VERSION="{}"'.format(args.version)) - print('export ESPURNA_RELEASE_DESTINATION="{}"'.format(args.destination)) - print('trap "ls -l ${ESPURNA_RELEASE_DESTINATION}" EXIT') + print('export ESPURNA_BUILD_DESTINATION="{}"'.format(args.destination)) + print("export ESPURNA_BUILD_SINGLE_SOURCE={}".format(int(args.single_source))) + if args.full_version: + print('export ESPURNA_BUILD_FULL_VERSION="{}"'.format(args.full_version)) + else: + if args.version: + print('export ESPURNA_BUILD_VERSION="{}"'.format(args.version)) + if args.suffix: + print('export ESPURNA_BUILD_REVISION="{}"'.format(args.revision)) + if args.suffix: + print('export ESPURNA_BUILD_VERSION_SUFFIX="{}"'.format(args.suffix)) + print('trap "ls -R ${ESPURNA_BUILD_DESTINATION}" EXIT') print( 'echo "Selected thread #{} out of {}"'.format( builder_thread + 1, builder_total_threads diff --git a/code/scripts/pio_main.py b/code/scripts/pio_main.py index a9956165..75b88efc 100644 --- a/code/scripts/pio_main.py +++ b/code/scripts/pio_main.py @@ -13,10 +13,10 @@ from espurna_utils import ( check_printsize, remove_float_support, ldscripts_inject_libpath, - app_inject_revision, + app_inject_version, dummy_ets_printf, app_inject_flags, - copy_release, + app_add_target_build_and_copy ) @@ -46,11 +46,11 @@ if "DISABLE_POSTMORTEM_STACKDUMP" in env["CPPFLAGS"]: "$BUILD_DIR/FrameworkArduino/core_esp8266_postmortem.cpp.o", dummy_ets_printf ) -# when using git, add -DAPP_REVISION=(git-commit-hash) -app_inject_revision(projenv) +# handle add -DAPP_VERSION=... that was set and / or detected +app_inject_version(projenv) # handle OTA board and flags here, since projenv is not available in pre-scripts app_inject_flags(projenv) -# handle `-t release` when CI does a tagged build -env.AlwaysBuild(env.Alias("release", "${BUILD_DIR}/${PROGNAME}.bin", copy_release)) +# handle when CI does a tagged build or user explicitly asked to store the firmware.bin +app_add_target_build_and_copy(projenv) diff --git a/code/scripts/pio_pre.py b/code/scripts/pio_pre.py index 395db20f..f67ab074 100644 --- a/code/scripts/pio_pre.py +++ b/code/scripts/pio_pre.py @@ -19,8 +19,7 @@ import sys from SCons.Script import ARGUMENTS -from espurna_utils.release import merge_cpp - +from espurna_utils.build import merge_cpp CI = "true" == os.environ.get("CI") PIO_PLATFORM = env.PioPlatform() @@ -112,14 +111,19 @@ if ESPURNA_OTA_PORT: else: env.Replace(UPLOAD_PROTOCOL="esptool") -# handle `-t release` parameters -if CI: - env.Append( - ESPURNA_RELEASE_REVISION=os.environ.get("ESPURNA_RELEASE_REVISION", ""), - ESPURNA_RELEASE_NAME=os.environ.get("ESPURNA_RELEASE_NAME", ""), - ESPURNA_RELEASE_VERSION=os.environ.get("ESPURNA_RELEASE_VERSION", ""), - ESPURNA_RELEASE_DESTINATION=os.environ.get("ESPURNA_RELEASE_DESTINATION", ""), - ) +# handle `-t build-and-copy` parameters +env.Append( + # what is the name suffix of the .bin + ESPURNA_BUILD_NAME=os.environ.get("ESPURNA_BUILD_NAME", ""), + # where to copy the resulting .bin + ESPURNA_BUILD_DESTINATION=os.environ.get("ESPURNA_BUILD_DESTINATION", ""), + # set the full string for the build, no need to change individual parts + ESPURNA_BUILD_FULL_VERSION=os.environ.get("ESPURNA_BUILD_FULL_VERSION", ""), + # or, replace parts of the version string that would've been auto-detected + ESPURNA_BUILD_VERSION=os.environ.get("ESPURNA_BUILD_VERSION", ""), + ESPURNA_BUILD_REVISION=os.environ.get("ESPURNA_BUILD_REVISION", ""), + ESPURNA_BUILD_VERSION_SUFFIX=os.environ.get("ESPURNA_BUILD_VERSION_SUFFIX", ""), +) # updates arduino core git to the latest master commit if CI: