PIO: fix env definitions (again) (#2212)

* base envs

* CI: speed up release process (since we still want some .bin files)

* fixup! CI: speed up release process (since we still want some .bin files)

* release dry run

* fixup! release dry run

* adjust

* fix .bin name

* it works

* minor cleanup for current git

* use pio suggestion about ldscript, reduce build_flags impact

* fix linker

* parse ${vars} instead of ignoring them

* add filtering and override file (sneak peak into tasmota's pio config)

* don't generate secure client (for now)

* formatting

* codacy

code/scripts/espurna_utils/__init__.py

@@ -25,6 +25,7 @@ from .ldscripts import ldscripts_inject_libpath
 from .lwip import lwip_inject_patcher
 from .postmortem import dummy_ets_printf
 from .git import app_inject_revision
+from .release import copy_release
 from .flags import app_inject_flags
 
 __all__ = [
@@ -36,4 +37,5 @@ __all__ = [
     "dummy_ets_printf",
     "app_inject_revision",
     "app_inject_flags",
+    "copy_release",
 ]

code/scripts/espurna_utils/release.py

@@ -0,0 +1,22 @@
+import os
+import shutil
+
+def copy_release(target, source, env):
+    # target filename and subdir for release files
+    name = env["ESPURNA_NAME"]
+    version = env["ESPURNA_VERSION"]
+
+    if not name or not version:
+        raise ValueError("Cannot set up release without release variables present")
+
+    destdir = os.path.join(env.subst("$PROJECT_DIR"), "..", "firmware", version)
+    if not os.path.exists(destdir):
+        os.makedirs(destdir)
+
+    dest = os.path.join(
+        destdir, "espurna-{version}-{name}.bin".format(version=version, name=name)
+    )
+    src = env.subst("$BUILD_DIR/${PROGNAME}.bin")
+
+    shutil.copy(src, dest)
+

code/scripts/generate_release_sh.py

@@ -0,0 +1,138 @@
+import os
+import argparse
+import re
+import shlex
+import configparser
+import collections
+
+CI = any([os.environ.get("TRAVIS"), os.environ.get("CI")])
+Build = collections.namedtuple("Build", "env extends build_flags src_build_flags")
+
+
+def expand_variables(cfg, value):
+    RE_VARS = re.compile("\$\{.*?\}")
+
+    for var in RE_VARS.findall(value):
+        section, option = var.replace("${", "").replace("}", "").split(".", 1)
+        value = value.replace(var, expand_variables(cfg, cfg.get(section, option)))
+
+    return value
+
+
+def get_builds(cfg):
+    RE_NEWLINE = re.compile("\r\n|\n")
+    BASE_BUILD_FLAGS = set(
+        shlex.split(expand_variables(cfg, cfg.get("env", "build_flags")))
+    )
+
+    for section in cfg.sections():
+        if (not section.startswith("env:")) or (
+            section.startswith("env:esp8266-") and section.endswith("-base")
+        ):
+            continue
+
+        build_flags = None
+        src_build_flags = None
+
+        try:
+            build_flags = cfg.get(section, "build_flags")
+            build_flags = RE_NEWLINE.sub(" ", build_flags).strip()
+            build_flags = " ".join(
+                BASE_BUILD_FLAGS ^ set(shlex.split(expand_variables(cfg, build_flags)))
+            )
+        except configparser.NoOptionError:
+            pass
+
+        try:
+            src_build_flags = cfg.get(section, "src_build_flags")
+            src_build_flags = RE_NEWLINE.sub(" ", src_build_flags).strip()
+            src_build_flags = expand_variables(cfg, src_build_flags)
+        except configparser.NoOptionError:
+            pass
+
+        yield Build(
+            section.replace("env:", ""),
+            cfg.get(section, "extends").replace("env:", ""),
+            build_flags,
+            src_build_flags,
+        )
+
+
+def find_any(string, values):
+    for value in values:
+        if value in string:
+            return True
+
+    return False
+
+
+def generate_lines(builds, ignore):
+    cores = []
+    generic = []
+
+    for build in builds:
+        if find_any(build.env, ignore):
+            continue
+
+        flags = []
+        if build.build_flags:
+            flags.append('PLATFORMIO_BUILD_FLAGS="{}"'.format(build.build_flags))
+        if build.src_build_flags:
+            flags.append('ESPURNA_FLAGS="{}"'.format(build.src_build_flags))
+        flags.append('ESPURNA_NAME="{env}"'.format(env=build.env))
+
+        cmd = ["env"]
+        cmd.extend(flags)
+        cmd.extend(["pio", "run", "-e", build.extends, "-s", "-t", "release"])
+
+        line = " ".join(cmd)
+
+        # push core variants to the front as they definetly include global build_flags
+        output = generic
+        if "ESPURNA_CORE" in build.src_build_flags:
+            output = cores
+
+        output.append(line)
+
+    return cores + generic
+
+
+def every(seq, nth, total):
+    index = 0
+    for value in seq:
+        if index == nth:
+            yield value
+        index = (index + 1) % total
+
+
+if __name__ == "__main__":
+    if not CI:
+        raise ValueError("* Not in CI *")
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("version")
+    parser.add_argument("--ignore", action="append")
+    args = parser.parse_args()
+
+    Config = configparser.ConfigParser()
+    with open("platformio.ini", "r") as f:
+        Config.read_file(f)
+
+    builder_total_threads = int(os.environ["BUILDER_TOTAL_THREADS"])
+    builder_thread = int(os.environ["BUILDER_THREAD"])
+    if builder_thread >= builder_total_threads:
+        raise ValueError("* Builder thread index out of range *")
+
+    builds = every(get_builds(Config), builder_thread, builder_total_threads)
+
+    print("#!/bin/bash")
+    print("set -e -x")
+    print('export ESPURNA_VERSION="{}"'.format(args.version))
+    print('trap "ls -l ${TRAVIS_BUILD_DIR}/firmware/${ESPURNA_VERSION}" EXIT')
+    print(
+        'echo "Selected thread #{} out of {}"'.format(
+            builder_thread + 1, builder_total_threads
+        )
+    )
+    for line in generate_lines(builds, args.ignore or ()):
+        print(line)

code/scripts/pio_main.py

@@ -17,6 +17,7 @@ from espurna_utils import (
     app_inject_revision,
     dummy_ets_printf,
     app_inject_flags,
+    copy_release,
 )
 
 Import("env", "projenv")
@@ -50,3 +51,6 @@ app_inject_revision(projenv)
 
 # handle OTA board and flags here, since projenv is not available in pre-scripts
 app_inject_flags(projenv)
+
+# handle `-t release` when CI does a tagged build
+env.AlwaysBuild(env.Alias("release", "${BUILD_DIR}/${PROGNAME}.bin", copy_release))

code/scripts/pio_pre.py

@@ -94,9 +94,11 @@ def ensure_platform_updated():
         log("Warning: no connection, cannot check for outdated packages", verbose=True)
 
 
-# handle OTA uploads
-# using env instead of ini to fix platformio ini changing hash on every change
+# handle build flags through os environment.
+# using env instead of ini to avoid platformio ini changing hash on every change
 env.Append(
+    ESPURNA_VERSION=os.environ.get("ESPURNA_VERSION", ""),
+    ESPURNA_NAME=os.environ.get("ESPURNA_NAME", ""),
     ESPURNA_BOARD=os.environ.get("ESPURNA_BOARD", ""),
     ESPURNA_AUTH=os.environ.get("ESPURNA_AUTH", ""),
     ESPURNA_FLAGS=os.environ.get("ESPURNA_FLAGS", "")
@@ -110,12 +112,13 @@ if ESPURNA_OTA_PORT:
 else:
     env.Replace(UPLOAD_PROTOCOL="esptool")
 
-# latest toolchain is still optional with PIO (TODO: recheck after 2.6.0!)
-# also updates arduino core git to the latest master commit
-if TRAVIS and (
-    env.GetProjectOption("platform") == CONFIG.get("common", "arduino_core_git")
-):
-    ensure_platform_updated()
+# updates arduino core git to the latest master commit
+if TRAVIS:
+    package_overrides = env.GetProjectOption("platform_packages")
+    for package in package_overrides:
+        if "https://github.com/esp8266/Arduino.git" in package:
+            ensure_platform_updated()
+            break
 
 # to speed-up build process, install libraries in either global or local shared storage
 if os.environ.get("ESPURNA_PIO_SHARED_LIBRARIES"):