diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 00000000..805734f7 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,697 @@ +# GitHub Actions Workflows Documentation + +This document provides a overview of all GitHub Actions workflows in the OpenMQTTGateway project. + +--- + +## Architecture Overview + +The workflow system is organized in two layers: + +### **Main Workflows** (User-facing triggers) +Entry points triggered by user actions, schedules, or events: +- `build.yml` - CI validation on push/PR +- `build_and_docs_to_dev.yml` - Daily development builds +- `release.yml` - Production releases +- `manual_docs.yml` - Documentation deployment +- `lint.yml` - Code formatting check +- `stale.yml` - Issue management + +### **Task Workflows** (Reusable components) +Parameterized building blocks called by main workflows: +- `task-build.yml` - Configurable firmware build +- `task-docs.yml` - Configurable documentation build +- `task-lint.yml` - Configurable code formatting check + +--- + +## Workflow Overview Table + +| Workflow | Trigger | Purpose | Artifacts | +|----------|---------|---------|-----------| +| `build.yml` | Push, Pull Request | CI Build Validation | Firmware binaries (7 days) | +| `build_and_docs_to_dev.yml` | Daily Cron, Manual | Development Builds + Docs | Firmware + Docs deployment | +| `release.yml` | Release Published | Production Release | Release assets + Docs | +| `manual_docs.yml` | Manual, Workflow Call | Documentation Only | GitHub Pages docs | +| `lint.yml` | Push, Pull Request | Code Format Check | None | +| `stale.yml` | Daily Cron | Issue Management | None | +| **`task-build.yml`** | **Workflow Call** | **Reusable Build Logic** | **Configurable** | +| **`task-docs.yml`** | **Workflow Call** | **Reusable Docs Logic** | **GitHub Pages** | +| **`task-lint.yml`** | **Workflow Call** | **Reusable Lint Logic** | **None** | + +--- + +## Detailed Workflow Documentation + +### 1. `build.yml` - Continuous Integration Build + +**Purpose**: Validates that code changes compile successfully across all supported hardware platforms. + +**Triggers**: +- **Push**: Every commit pushed to any branch +- **Pull Request**: Every PR creation or update + +**What it does**: +1. **Build job**: Calls `task-build.yml` with CI parameters + - Builds firmware for **83 hardware environments** in parallel +2. **Documentation job**: Inline job that validates docs build (doesn't deploy) + - Downloads common config from theengs.io + - Runs `npm install` and `npm run docs:build` + - Uses Node.js 14.x + +**Technical Details**: +- **Calls**: `task-build.yml` only (documentation is inline) +- Python version: 3.13 (for build job) +- Build strategy: Parallel matrix via task workflow +- Artifact retention: 7 days +- Development OTA: Disabled (`enable-dev-ota: false`) + +**Outputs**: +- Firmware binaries for each environment (83 artifacts) +- No documentation deployment (validation only) + +**Use Case**: Ensures no breaking changes before merge. Fast feedback for developers. + +**Execution Context**: Runs for ALL contributors on ALL branches. + +--- + +### 2. `build_and_docs_to_dev.yml` - Development Deployment Pipeline + +**Purpose**: Creates nightly development builds and deploys documentation to the `/dev` subdirectory for testing. + +**Triggers**: +- **Schedule**: Daily at midnight UTC (`0 0 * * *`) +- **Manual**: Via workflow_dispatch button + +**What it does**: +1. **Prepare job**: Generates 6-character short SHA +2. **Build job**: Calls `task-build.yml` with development parameters + - Builds firmware for **83 hardware environments** in parallel + - Enables development OTA updates with SHA commit version +3. **Deploy job**: Prepares and uploads assets + - Creates library dependency zips for each board + - Generates source code zip + - Removes test environment binaries +4. **Documentation job**: Calls `task-docs.yml` with development parameters + - Deploys to `/dev` subdirectory + - Adds "DEVELOPMENT" watermark + - Runs PageSpeed Insights + +**Technical Details**: +- **Calls**: `task-build.yml` + `task-docs.yml` +- Python version: 3.13 (build), 3.11 (docs) +- Node.js version: 16.x (docs) +- Repository restriction: Hardcoded to `1technophile` owner only +- Artifact retention: 1 day +- Build flag: `enable-dev-ota: true` (passed to task-build.yml) + +**Outputs**: +- Firmware binaries with `-firmware.bin` suffix +- Bootloader and partition binaries +- Library dependency zips per board +- Source code zip +- Documentation deployed to `docs.openmqttgateway.com/dev/` + +**Version Labeling**: +- Git SHA (6 chars) injected into firmware +- Docs tagged: "DEVELOPMENT SHA:XXXXXX TEST ONLY" + +**Use Case**: Daily bleeding-edge builds for early adopters and testing. Preview documentation changes. + +**Execution Context**: Only runs on `1technophile` repository owner. Forks will skip this workflow automatically. + +--- + +### 3. `release.yml` - Production Release Pipeline + +**Purpose**: Creates official release builds when a new version is published. + +**Triggers**: +- **Release**: When a GitHub release is published (tagged) + +**What it does**: +1. **Prepare job**: Extracts version tag and release info +2. **Build job**: Calls `task-build.yml` with production parameters + - Builds firmware for **83 hardware environments** in parallel + - Injects release tag version into firmware +3. **Deploy job**: Prepares and uploads release assets + - Downloads all build artifacts + - Reorganizes for `prepare_deploy.sh` script + - Creates library zips and source zip + - Uploads to GitHub Release +4. **Documentation job**: Calls `task-docs.yml` for production docs + +**Technical Details**: +- **Calls**: `task-build.yml` + `task-docs.yml` +- Python version: 3.13 (build), 3.11 (docs) +- Node.js version: 18.x +- Build flag: Standard (no DEVELOPMENTOTA) +- Artifact retention: 90 days +- Uses `prepare_deploy.sh` script for asset preparation + +**Outputs**: +- Production firmware binaries attached to GitHub Release +- Library zips per board +- Source code zip +- Production documentation at `docs.openmqttgateway.com/` + +**Version Labeling**: +- Git tag (e.g., `v1.2.3`) injected into firmware and `latest_version.json` + +**Workflow Chain**: +``` +prepare → build (task-build.yml) → deploy → documentation (task-docs.yml) +``` + +**Use Case**: Official releases for end users. Stable, versioned firmware. + +**Execution Context**: Triggered by repository maintainers creating releases. + +--- + +### 4. `manual_docs.yml` - Documentation Deployment + +**Purpose**: Entry point for standalone documentation deployment to GitHub Pages. + +**Triggers**: +- **Manual**: Via workflow_dispatch button +- **Workflow Call**: Can be called by other workflows (legacy compatibility) + +**What it does**: +1. Calls `task-docs.yml` with production parameters +2. Deploys to root directory (`/`) of GitHub Pages + +**Technical Details**: +- **Calls**: `task-docs.yml` +- Python version: 3.11 +- Node.js version: 14.x +- Version source: Latest GitHub release tag +- WebUploader manifest: Enabled + +**Outputs**: +- Production documentation at `docs.openmqttgateway.com/` +- Custom domain: `docs.openmqttgateway.com` (via CNAME) + +**Use Case**: Standalone documentation updates without full release process. + +**Execution Context**: Manual trigger or legacy workflow calls. + +--- + +### 5. `lint.yml` - Code Format Validation + +**Purpose**: Ensures code follows consistent formatting standards. + +**Triggers**: +- **Push**: Every commit pushed to any branch +- **Pull Request**: Every PR creation or update + +**What it does**: +1. Calls `task-lint.yml` with specific parameters +2. Checks formatting in `./main` directory only + +**Technical Details**: +- **Calls**: `task-lint.yml` +- clang-format version: 9 +- File extensions: `.h`, `.ino` (not `.cpp`) +- Source directory: `main` (single directory) + +**Configuration**: +```yaml +source: 'main' +extensions: 'h,ino' +clang-format-version: '9' +``` + +**Use Case**: Maintains code quality and consistency. Prevents formatting debates in PRs. + +**Execution Context**: Runs for ALL contributors on ALL branches. + +--- + +### 6. `stale.yml` - Issue and PR Management + +**Purpose**: Automatically closes inactive issues and pull requests to reduce maintenance burden. + +**Triggers**: +- **Schedule**: Daily at 00:30 UTC (`30 0 * * *`) + +**What it does**: +1. Marks issues/PRs as stale after 90 days of inactivity +2. Closes stale issues/PRs after 14 additional days +3. Exempts issues labeled "enhancement" + +**Configuration**: +- Stale after: 90 days +- Close after: 14 days (104 days total) +- Stale label: `stale` +- Exempt labels: `enhancement` + +**Messages**: +- Stale: "This issue is stale because it has been open for 90 days with no activity." +- Close: "This issue was closed because it has been inactive for 14 days since being marked as stale." + +**Use Case**: Housekeeping. Reduces backlog of abandoned issues. + +**Execution Context**: Automated maintenance by GitHub bot. + +--- + +## Task Workflows (Reusable Components) + +### 7. `task-build.yml` - Reusable Build Workflow + +**Purpose**: Parameterized firmware build logic used by multiple workflows. + +**Trigger**: `workflow_call` only (called by other workflows) + +**Parameters**: +- `python-version`: Python version (default: '3.13') +- `enable-dev-ota`: Enable development OTA (default: false) +- `version-tag`: Version to inject (default: 'unspecified') +- `artifact-retention-days`: Artifact retention (default: 7) +- `artifact-name-prefix`: Artifact name prefix (default: '') +- `prepare-for-deploy`: Prepare for deployment (default: false) + +**What it does**: +1. **Load environments**: Reads environment list from `environments.json` +2. **Matrix build**: Builds all 83 environments in parallel +3. **Build execution**: Calls unified `ci.sh build [OPTIONS]`: + - ``: Target hardware (e.g., `esp32dev-ble`) + - `--version `: Version to inject (SHA for dev, tag for prod) + - `--mode `: Build mode (enables/disables OTA) + - `--deploy-ready`: Prepare artifacts for deployment + - `--output `: Output directory for artifacts (default: `generated/artifacts/`) + +**Command Flow**: +```bash +./scripts/ci.sh build esp32dev-ble --version v1.8.0 --mode prod --deploy-ready + ↓ + ├─→ ci_build.sh (orchestrator) + │ ├─→ ci_set_version.sh v1.8.0 [--dev] + │ ├─→ ci_build_firmware.sh esp32dev-ble [--dev-ota] + │ └─→ ci_prepare_artifacts.sh esp32dev-ble [--deploy] → outputs to generated/artifacts/ +``` + +**Technical Details**: +- Runs on: Ubuntu latest +- PlatformIO version: 6.1.18 (custom fork: `pioarduino/platformio-core`) +- Python package manager: `uv` (astral-sh/setup-uv@v6) +- Strategy: Matrix with fail-fast: false +- Main orchestrator: `ci.sh` → `ci_build.sh` → sub-scripts + +**Callers**: +- `build.yml` (CI validation) +- `build_and_docs_to_dev.yml` (development builds) +- `release.yml` (production releases) + +--- + +### 8. `task-docs.yml` - Reusable Documentation Workflow + +**Purpose**: Parameterized documentation build and deployment logic. + +**Trigger**: `workflow_call` only (called by other workflows) + +**Parameters**: +- `python-version`: Python version (default: '3.11') +- `node-version`: Node.js version (default: '14.x') +- `version-source`: Version source ('release', 'git-tag', 'custom') +- `custom-version`: Custom version string (optional) +- `base-path`: Base URL path (default: '/') +- `destination-dir`: Deploy directory (default: '.') +- `generate-webuploader`: Generate WebUploader manifest (default: true) +- `webuploader-args`: WebUploader generation arguments (optional) +- `run-pagespeed`: Run PageSpeed Insights (default: false) +- `pagespeed-url`: URL for PageSpeed test (optional) + +**What it does**: +1. **Build documentation**: Calls unified `ci.sh site [OPTIONS]`: + - `--mode `: Documentation mode + - `--custom-version `: Custom version string + - `--version-source `: Version source + - `--url-prefix `: Base URL path (e.g., `/dev/`) + - `--webuploader-args `: WebUploader options + - `--no-webuploader`: Skip manifest generation +2. **Deploy**: Publishes to GitHub Pages using `peaceiris/actions-gh-pages@v3` +3. **PageSpeed test**: Optionally runs performance audit + +**Command Flow**: +```bash +./scripts/ci.sh site --mode prod --version-source release --url-prefix / + ↓ + └─→ ci_site.sh (orchestrator) + ├─→ generate_board_docs.py (auto-generate board docs) + ├─→ npm run docs:build (VuePress compilation) + └─→ gen_wu.py (WebUpdater manifest) +``` + +**Callers**: +- `build_and_docs_to_dev.yml` (dev docs to `/dev`) +- `release.yml` (production docs to `/`) +- `manual_docs.yml` (manual production docs) + +--- + +### 9. `task-lint.yml` - Reusable Lint Workflow + +**Purpose**: Parameterized code formatting validation. + +**Trigger**: `workflow_call` only (called by other workflows) + +**Parameters**: +- `source`: Source directories to lint (default: './lib ./main') +- `extensions`: File extensions to check (default: 'h,ino,cpp') +- `clang-format-version`: clang-format version (default: '9') +- `exclude-pattern`: Pattern to exclude (optional) + +**What it does**: +1. Checks out code +2. Installs clang-format (specified version) +3. Runs unified `ci.sh qa [OPTIONS]`: + - `--check`: Validation mode (exit on violations) + - `--fix`: Auto-fix formatting issues + - `--source `: Directory to lint + - `--extensions `: File extensions (comma-separated) + - `--clang-format-version `: Formatter version +4. Fails if formatting violations found + +**Command Flow**: +```bash +./scripts/ci.sh qa --check --source main --extensions h,ino --clang-format-version 9 + ↓ + └─→ ci_qa.sh (formatter) + └─→ clang-format (checks/fixes code style) +``` + +**Technical Details**: +- Script: `ci_qa.sh` (custom formatting check script) +- Install: `clang-format-$version` via apt-get +- Default source: `main` (single directory) +- Default extensions: `h,ino` (not cpp) + +**Callers**: +- `lint.yml` (CI lint check) + +**Default Behavior**: If called without parameters, lints `main` directory for `.h` and `.ino` files. + +--- + +## Workflow Dependencies and Call Chain + +```mermaid +flowchart TD + %% Triggers + subgraph triggers ["🎯 Triggers"] + push["Push / Pull Request"] + release["Release Published"] + manual["Manual Trigger"] + cron1["Cron: Daily 00:00 UTC"] + cron2["Cron: Daily 00:30 UTC"] + end + +subgraph github_workflows ["📋 GitHub Workflows"] + %% Main Workflows + subgraph main ["📋 Main Workflows"] + lint["lint.yml
Format Check"] + + build["build.yml
CI Build"] + + release_wf["release.yml
Production Release"] + manual_docs["manual_docs.yml
Docs Only"] + build_dev["build_and_docs_to_dev.yml
Dev Builds"] + stale["stale.yml
Issue Management"] + end + + %% Task Workflows + subgraph tasks ["⚙️ Task Workflows"] + task_build["task-build.yml
Build Firmware"] + task_docs["task-docs.yml
Build & Deploy Docs"] + task_lint["task-lint.yml
Code Format"] + end +end + +subgraph ci_scripts ["🔧 CI Scripts"] + %% CI Scripts Layer + subgraph bash ["🔧 Orchestrator"] + ci_main["ci.sh
(main dispatcher)"] + ci_build_script["ci_build.sh
(build orchestrator)"] + ci_site_script["ci_site.sh
(docs orchestrator)"] + ci_qa_script["ci_qa.sh
(lint orchestrator)"] + end + + %% Sub-Scripts Layer + subgraph sub_scripts ["⚙️ Workers"] + ci_set_ver["ci_set_version.sh
(version injection)"] + ci_build_fw["ci_build_firmware.sh
(PlatformIO build)"] + ci_prep_art["ci_prepare_artifacts.sh
(artifact packaging)"] + gen_board["generate_board_docs.py
(board docs)"] + gen_wu["gen_wu.py
(WebUpdater manifest)"] + clang_fmt["clang-format
(code formatter)"] + end +end + + %% Trigger connections + push --> lint + push --> build + + release --> release_wf + manual --> manual_docs + cron1 --> build_dev + cron2 --> stale + + %% Main workflow to task workflow connections + build -->|calls| task_build + lint -->|calls| task_lint + build_dev -->|calls| task_build + build_dev -->|calls| task_docs + release_wf -->|calls| task_build + release_wf -->|calls| task_docs + manual_docs -->|calls| task_docs + + %% Task workflows to CI scripts + task_build -->|"ci.sh build
--version --mode
--deploy-ready"| ci_main + task_docs -->|"ci.sh site
--mode --url-prefix
--version-source"| ci_main + task_lint -->|"ci.sh qa
--check --source
--extensions"| ci_main + + %% CI main dispatcher to orchestrators + ci_main -->|"route: build"| ci_build_script + ci_main -->|"route: site"| ci_site_script + ci_main -->|"route: qa"| ci_qa_script + + %% Orchestrators to workers + ci_build_script --> ci_set_ver + ci_build_script --> ci_build_fw + ci_build_script --> ci_prep_art + + ci_site_script --> gen_board + ci_site_script --> gen_wu + + ci_qa_script --> clang_fmt + + %% Styling + classDef triggerStyle fill:#e1f5ff,stroke:#0066cc,stroke-width:2px + classDef mainStyle fill:#fff4e6,stroke:#ff9900,stroke-width:2px + classDef taskStyle fill:#e6f7e6,stroke:#00aa00,stroke-width:2px + classDef ciStyle fill:#ffe6f0,stroke:#cc0066,stroke-width:2px + classDef subStyle fill:#f0e6ff,stroke:#9933ff,stroke-width:2px + + class push,release,manual,cron1,cron2 triggerStyle + class build,lint,release_wf,manual_docs,build_dev,stale mainStyle + class task_build,task_docs,task_lint taskStyle + class ci_main,ci_build_script,ci_site_script,ci_qa_script ciStyle + class ci_set_ver,ci_build_fw,ci_prep_art,gen_board,gen_wu,clang_fmt subStyle + + + style github_workflows stroke:#6A7BD8,stroke-dasharray:6 4,stroke-width:1.8px,fill:#fbfbfc + style main stroke:#6A7BD8,stroke-dasharray:6 4,stroke-width:0.6px,fill:#fcfdff + style tasks stroke:#6A7BD8,stroke-dasharray:6 4,stroke-width:0.6px,fill:#fcfdff + + style ci_scripts stroke:#FF9A3C,stroke-dasharray:6 4,stroke-width:1.8px,fill:#fffaf5 + style bash stroke:#FF9A3C,stroke-dasharray:6 4,stroke-width:0.6px,fill:#fffaf5 + style sub_scripts stroke:#FF9A3C,stroke-dasharray:6 4,stroke-width:0.6px,fill:#fffaf5 + + style triggers fill:none,stroke:none + +``` + +### Workflow Relationships + +**Main → Task Mapping**: +- `build.yml` → calls `task-build.yml` (also contains inline documentation job) +- `lint.yml` → calls `task-lint.yml` +- `build_and_docs_to_dev.yml` → calls `task-build.yml` + `task-docs.yml` (with prepare/deploy jobs) +- `release.yml` → calls `task-build.yml` + `task-docs.yml` (with prepare/deploy jobs) +- `manual_docs.yml` → calls `task-docs.yml` +- `stale.yml` → standalone (no dependencies) + +**Task → CI Script Mapping**: +- `task-build.yml` → `ci.sh build --version --mode --deploy-ready` + - Routes to: `ci_build.sh` → `ci_set_version.sh`, `ci_build_firmware.sh`, `ci_prepare_artifacts.sh` + - Output: `generated/artifacts/` (default, can be overridden with `--output`) +- `task-docs.yml` → `ci.sh site --mode --version-source --url-prefix --webuploader-args` + - Routes to: `ci_site.sh` → `generate_board_docs.py`, `gen_wu.py`, VuePress + - Output: `generated/site/` +- `task-lint.yml` → `ci.sh qa --check --source --extensions --clang-format-version` + - Routes to: `ci_qa.sh` → `clang-format` + +**Job Dependencies**: +- `build_and_docs_to_dev.yml`: prepare → build (task) → deploy & documentation (task) +- `release.yml`: prepare → build (task) → deploy → documentation (task) + +**Script Execution Flow**: +``` +GitHub Action (task-*.yml) + ↓ +./scripts/ci.sh [OPTIONS] ← Main dispatcher + ↓ +./scripts/ci_.sh ← Command orchestrator + ↓ +./scripts/ci_*.sh / *.py ← Worker scripts +``` + +--- + +## Environment Configuration + +### Centralized Environment Management + +All build environments are defined in `.github/workflows/environments.json`: + +```json +{ + "environments": { + "all": [ ...83 environments ], + "metadata": { + "totalCount": 83, + "lastUpdated": "2024-12-25", + "categories": { + "esp32": 50, + "esp8266": 20, + "specialized": 13 + } + } + } +} +``` + +**Benefits**: +- ✅ Single source of truth +- ✅ Eliminates duplication across workflows +- ✅ Easier to maintain and update +- ✅ Consistent builds across CI/dev/release + +### Environment Categories + +**ESP32 Family** (~50 environments): +- Standard: `esp32dev-*` variants +- ESP32-S3: `esp32s3-*` variants +- ESP32-C3: `esp32c3-*` variants +- Specialized boards: M5Stack, Heltec, LilyGO, Theengs + +**ESP8266 Family** (~20 environments): +- NodeMCU: `nodemcuv2-*` variants +- Sonoff: `sonoff-*` variants +- Generic: `esp8266-*` variants + +**Specialized Boards** (~13 environments): +- Theengs Plug +- Theengs Bridge +- RF Bridge variants +- Custom board configurations + +--- + +## Configuration Variables + +### Repository Restrictions + +**Development Builds** (`build_and_docs_to_dev.yml`): +- Hardcoded restriction: `github.repository_owner == '1technophile'` +- Only runs for the main repository owner +- Prevents accidental deployments from forks +- No configuration variable needed + +**Release Builds** (`release.yml`): +- No repository restrictions +- Runs on any fork when a release is published +- Deploy step requires proper GitHub token permissions + +**Documentation** (`manual_docs.yml`): +- No repository restrictions +- Can be triggered manually from any fork +- Requires GitHub Pages to be configured + +--- + +## Glossary + +- **Environment**: A specific hardware board + gateway combination (e.g., `esp32dev-ble`) +- **Matrix Build**: Parallel execution of builds across multiple environments +- **Artifact**: Build output stored temporarily for download (firmware binaries) +- **workflow_call**: GitHub Actions feature for calling one workflow from another +- **workflow_dispatch**: Manual trigger button for workflows +- **Task Workflow**: Reusable workflow component with parameterized inputs +- **Main Workflow**: Entry point workflow triggered by events or schedules +- **CNAME**: Custom domain configuration for GitHub Pages +- **OTA**: Over-The-Air firmware update capability +- **SHA**: Git commit hash used for version identification in dev builds + +--- + +## Maintenance Notes + +### CI/CD Script Architecture + +**Main Entry Point**: `ci.sh` (unified interface) +- Commands: `build`, `site`, `qa`, `all` +- Routes to specialized orchestrators +- Provides consistent CLI across all operations + +**Build System** (`ci.sh build`): +- PlatformIO 6.1.18 from custom fork: `pioarduino/platformio-core` +- Python package manager: `uv` for fast dependency installation +- Orchestrator: `ci_build.sh` + - Worker: `ci_set_version.sh` (version injection) + - Worker: `ci_build_firmware.sh` (PlatformIO compilation) + - Worker: `ci_prepare_artifacts.sh` (artifact packaging) + +**Documentation System** (`ci.sh site`): +- Documentation framework: VuePress +- Orchestrator: `ci_site.sh` + - Worker: `generate_board_docs.py` (auto-generate board pages) + - Worker: `gen_wu.py` (WebUpdater manifest) + - External: Common config from theengs.io + +**Code Quality** (`ci.sh qa`): +- Orchestrator: `ci_qa.sh` + - Worker: `clang-format` version 9 + - Default scope: `main` directory, `.h` and `.ino` files + +**Configuration**: +- Environment list: `.github/workflows/environments.json` (83 environments) +- Task workflows: `task-*.yml` (reusable GitHub Actions components) +- Repository owner restriction: Hardcoded to `1technophile` for dev deployments +- All scripts located in: `./scripts/` + +**Local Development**: +```bash +# Build firmware locally +./scripts/ci.sh build esp32dev-ble --mode dev --version test + +# Build documentation locally +./scripts/ci.sh site --mode dev --preview + +# Check code format +./scripts/ci.sh qa --check + +# Run complete pipeline +./scripts/ci.sh all esp32dev-ble --version v1.8.0 +``` + +--- + +**Document Version**: 2.1 +**Last Updated**: December 29, 2025 +**Maintainer**: OpenMQTTGateway Development Team diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7ad46d4e..818e6cf9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,125 +4,17 @@ on: [push, pull_request] jobs: build: - strategy: - fail-fast: false - matrix: - environments: - - "rfbridge" - - "rfbridge-direct" - - "esp32dev-all-test" - - "esp32dev-rf" - - "esp32dev-pilight-cc1101" - - "esp32dev-somfy-cc1101" - - "esp32dev-pilight-somfy-cc1101" - - "esp32dev-weatherstation" - - "esp32dev-gf-sun-inverter" - - "esp32dev-ir" - - "esp32dev-ble" - - "esp32dev-ble-mqtt-undecoded" - - "esp32dev-ble-aws" - - "esp32feather-ble" - - "esp32-lolin32lite-ble" - - "esp32-olimex-gtw-ble-eth" - - "esp32-olimex-gtw-ble-poe" - - "esp32-olimex-gtw-ble-poe-iso" - - "esp32-wt32-eth01-ble-eth" - - "esp32-olimex-gtw-ble-wifi" - - "esp32-m5stick-ble" - - "esp32-m5stack-ble" - - "esp32-m5tough-ble" - - "esp32-m5stick-c-ble" - - "esp32-m5stick-cp-ble" - - "esp32s3-atomS3U" - - "esp32-m5atom-matrix" - - "esp32-m5atom-lite" - - "esp32dev-rtl_433" - - "esp32dev-rtl_433-fsk" - - "esp32doitv1-aithinker-r01-sx1278" - - "heltec-rtl_433" - - "heltec-rtl_433-fsk" - - "heltec-ble" - - "lilygo-rtl_433" - - "lilygo-rtl_433-fsk" - - "lilygo-ble" - - "esp32dev-multi_receiver" - - "esp32dev-multi_receiver-pilight" - - "tinypico-ble" - - "ttgo-lora32-v1" - - "ttgo-lora32-v21" - - "ttgo-t-beam" - - "heltec-wifi-lora-32" - - "shelly-plus1" - - "nodemcuv2-all-test" - - "nodemcuv2-fastled-test" - - "nodemcuv2-2g" - - "nodemcuv2-ir" - - "nodemcuv2-serial" - - "avatto-bakeey-ir" - - "nodemcuv2-rf" - - "nodemcuv2-rf-cc1101" - - "nodemcuv2-somfy-cc1101" - - "manual-wifi-test" - - "rf-wifi-gateway" - - "nodemcuv2-rf2" - - "nodemcuv2-rf2-cc1101" - - "nodemcuv2-pilight" - - "nodemcuv2-weatherstation" - - "sonoff-basic" - - "sonoff-basic-rfr3" - - "esp32dev-ble-datatest" - - "esp32s3-dev-c1-ble" - - "esp32c3-dev-m1-ble" - - "airm2m_core_esp32c3" - - "esp32c3_lolin_mini" - - "esp32c3-m5stamp" - - "thingpulse-espgateway" - - "theengs-bridge" - - "esp32dev-ble-idf" - - "theengs-bridge-v11" - - "theengs-plug" - - "esp32dev-ble-broker" - - "esp32s3-m5stack-stamps3" - - "esp32c3u-m5stamp" - - "lilygo-t3-s3-rtl_433" - - "lilygo-t3-s3-rtl_433-fsk" - runs-on: ubuntu-latest - name: Build with PlatformIO - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.13' - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - version: "latest" - enable-cache: false - - name: Install dependencies - run: | - uv pip install --system -U https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip - - name: Extract ESP32 platform version from platformio.ini - run: | - ESP32_VERSION=$(grep 'esp32_platform\s*=' platformio.ini | cut -d'@' -f2 | tr -d '[:space:]') - echo "ESP32_PLATFORM_VERSION=${ESP32_VERSION}" >> $GITHUB_ENV - - name: Run PlatformIO - env: - PYTHONIOENCODING: utf-8 - PYTHONUTF8: '1' - run: platformio run -e ${{ matrix.environments }} - - name: Upload Assets - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.environments }} - path: | - .pio/build/*/firmware.bin - .pio/build/*/partitions.bin - retention-days: 7 + name: Build firmware + uses: ./.github/workflows/task-build.yml + with: + python-version: '3.13' + enable-dev-ota: false + artifact-retention-days: 7 + prepare-for-deploy: false documentation: + name: Build documentation runs-on: ubuntu-latest - name: Create the documentation steps: - uses: actions/checkout@v4 - name: Set up Node.js diff --git a/.github/workflows/build_and_docs_to_dev.yml b/.github/workflows/build_and_docs_to_dev.yml index 3c2ea37c..7f698fd1 100644 --- a/.github/workflows/build_and_docs_to_dev.yml +++ b/.github/workflows/build_and_docs_to_dev.yml @@ -1,173 +1,65 @@ name: Build binaries, docs and publish to dev folder + on: workflow_dispatch: schedule: - cron: '0 0 * * *' + jobs: - build: - strategy: - fail-fast: false - matrix: - environments: - - "rfbridge" - - "rfbridge-direct" - - "theengs-bridge" - - "theengs-bridge-v11" - - "theengs-plug" - - "esp32dev-all-test" - - "esp32dev-rf" - - "esp32dev-pilight" - - "esp32dev-pilight-cc1101" - - "esp32dev-somfy-cc1101" - - "esp32dev-pilight-somfy-cc1101" - - "esp32dev-weatherstation" - - "esp32dev-gf-sun-inverter" - - "esp32dev-ir" - - "esp32dev-ble" - - "esp32dev-ble-broker" - - "esp32dev-ble-mqtt-undecoded" - - "esp32dev-ble-aws" - - "esp32feather-ble" - - "esp32-lolin32lite-ble" - - "esp32-olimex-gtw-ble-eth" - - "esp32-olimex-gtw-ble-poe" - - "esp32-olimex-gtw-ble-poe-iso" - - "esp32-wt32-eth01-ble-eth" - - "esp32-olimex-gtw-ble-wifi" - - "esp32-m5stick-ble" - - "esp32-m5stack-ble" - - "esp32-m5tough-ble" - - "esp32-m5stick-c-ble" - - "esp32-m5stick-cp-ble" - - "esp32-m5atom-matrix" - - "esp32-m5atom-lite" - - "esp32doitv1-aithinker-r01-sx1278" - - "esp32dev-rtl_433" - - "esp32dev-rtl_433-fsk" - - "heltec-rtl_433" - - "heltec-rtl_433-fsk" - - "heltec-ble" - - "lilygo-rtl_433" - - "lilygo-rtl_433-fsk" - - "lilygo-t3-s3-rtl_433" - - "lilygo-t3-s3-rtl_433-fsk" - - "lilygo-ble" - - "esp32dev-multi_receiver" - - "esp32dev-multi_receiver-pilight" - - "tinypico-ble" - - "ttgo-lora32-v1" - - "ttgo-lora32-v21" - - "ttgo-t-beam" - - "heltec-wifi-lora-32" - - "shelly-plus1" - - "nodemcuv2-all-test" - - "nodemcuv2-fastled-test" - - "nodemcuv2-2g" - - "nodemcuv2-ir" - - "nodemcuv2-serial" - - "avatto-bakeey-ir" - - "nodemcuv2-rf" - - "nodemcuv2-rf-cc1101" - - "nodemcuv2-somfy-cc1101" - - "manual-wifi-test" - - "rf-wifi-gateway" - - "nodemcuv2-rf2" - - "nodemcuv2-rf2-cc1101" - - "nodemcuv2-pilight" - - "nodemcuv2-weatherstation" - - "sonoff-basic" - - "sonoff-basic-rfr3" - - "esp32dev-ble-datatest" - - "esp32s3-dev-c1-ble" - - "esp32s3-m5stack-stamps3" - - "esp32s3-atomS3U" - - "esp32c3-dev-m1-ble" - - "airm2m_core_esp32c3" - - "esp32c3-dev-c2-ble" - - "esp32c3-dev-c2-ble-no-serial" - - "esp32c3_lolin_mini" - - "esp32c3_lolin_mini_with_serial" - - "esp32c3-m5stamp" - - "esp32c3u-m5stamp" - - "thingpulse-espgateway" - - "esp32dev-ble-idf" + prepare: runs-on: ubuntu-latest if: github.repository_owner == '1technophile' - name: Build ${{ matrix.environments }} + outputs: + short-sha: ${{ steps.short-sha.outputs.sha }} steps: - uses: actions/checkout@v4 - uses: benjlevesque/short-sha@v2.1 id: short-sha with: length: 6 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - version: "latest" - enable-cache: false - - name: Install dependencies - run: | - uv pip install --system -U https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip - - name: Set sha tag - run: | - sed -i "s/version_tag/${{ steps.short-sha.outputs.sha }}/g" main/User_config.h scripts/latest_version_dev.json - - name: Run PlatformIO - env: - PYTHONIOENCODING: utf-8 - PYTHONUTF8: '1' - run: | - export PLATFORMIO_BUILD_FLAGS="'-DDEVELOPMENTOTA=true'" - platformio run -e ${{ matrix.environments }} - - name: Prepare firmware artifacts - run: | - mkdir -p firmware - cp .pio/build/${{ matrix.environments }}/firmware.bin firmware/${{ matrix.environments }}-firmware.bin - if [ -f .pio/build/${{ matrix.environments }}/partitions.bin ]; then - cp .pio/build/${{ matrix.environments }}/partitions.bin firmware/${{ matrix.environments }}-partitions.bin - fi - if [ -f .pio/build/${{ matrix.environments }}/bootloader.bin ]; then - cp .pio/build/${{ matrix.environments }}/bootloader.bin firmware/${{ matrix.environments }}-bootloader.bin - fi - - name: Upload firmware - uses: actions/upload-artifact@v4 - with: - name: firmware-${{ matrix.environments }} - path: firmware/ - retention-days: 1 + + build: + needs: prepare + name: Build development firmware + uses: ./.github/workflows/task-build.yml + with: + python-version: '3.13' + enable-dev-ota: true + version-tag: ${{ needs.prepare.outputs.short-sha }} + artifact-retention-days: 1 + artifact-name-prefix: 'firmware-' + prepare-for-deploy: true deploy: - needs: build + needs: [prepare, build] runs-on: ubuntu-latest if: github.repository_owner == '1technophile' name: Deploy binaries and docs steps: - uses: actions/checkout@v4 - - uses: benjlevesque/short-sha@v2.1 - id: short-sha - with: - length: 6 + - name: Download all firmware artifacts uses: actions/download-artifact@v4 with: pattern: firmware-* path: toDeploy merge-multiple: true + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.13" + - name: Install uv uses: astral-sh/setup-uv@v6 with: version: "latest" enable-cache: false + - name: Install dependencies run: | uv pip install --system -U https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip + - name: Create library zips run: | # Install libraries for a representative environment to get libdeps @@ -180,6 +72,7 @@ jobs: zip -r "${i%/}-libraries.zip" "$i" done mv *.zip ../../toDeploy/ + - name: Prepare additional assets run: | cd toDeploy @@ -189,33 +82,22 @@ jobs: # Zip source code zip -r toDeploy/OpenMQTTGateway_sources.zip main LICENSE.txt ls -lA toDeploy/ - - name: Set sha tag for docs - run: | - sed -i "s/version_tag/DEVELOPMENT SHA:${{ steps.short-sha.outputs.sha }} TEST ONLY/g" docs/.vuepress/config.js - sed -i "s|base: '/'|base: '/dev/'|g" docs/.vuepress/config.js - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: "16.x" - - name: Download Common Config - run: | - curl -o docs/.vuepress/public/commonConfig.js https://www.theengs.io/commonConfig.js - - name: Build documentation - run: | - python ./scripts/gen_wu.py --dev - npm install - npm run docs:build - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./docs/.vuepress/dist - destination_dir: dev - cname: docs.openmqttgateway.com - - name: Running Page Speed Insights - uses: jakepartusch/psi-action@v1.3 - id: psi - with: - url: "https://docs.openmqttgateway.com/dev/" - threshold: 60 - key: ${{ secrets.APIKEY }} + + documentation: + needs: [prepare, build] + name: Build and deploy development documentation + uses: ./.github/workflows/task-docs.yml + with: + python-version: '3.11' + node-version: '16.x' + version-source: 'custom' + custom-version: 'DEVELOPMENT SHA:${{ needs.prepare.outputs.short-sha }} TEST ONLY' + base-path: '/dev/' + destination-dir: 'dev' + generate-webuploader: true + webuploader-args: '--dev' + run-pagespeed: true + pagespeed-url: 'https://docs.openmqttgateway.com/dev/' + secrets: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + APIKEY: ${{ secrets.APIKEY }} diff --git a/.github/workflows/environments.json b/.github/workflows/environments.json new file mode 100644 index 00000000..4f0b145b --- /dev/null +++ b/.github/workflows/environments.json @@ -0,0 +1,106 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "description": "Centralized list of all PlatformIO build environments for OpenMQTTGateway workflows", + "version": "1.0.0", + "lastUpdated": "2025-12-24", + "environments": { + "all": [ + "rfbridge", + "rfbridge-direct", + "theengs-bridge", + "theengs-bridge-v11", + "theengs-plug", + "esp32dev-all-test", + "esp32dev-rf", + "esp32dev-pilight", + "esp32dev-pilight-cc1101", + "esp32dev-somfy-cc1101", + "esp32dev-pilight-somfy-cc1101", + "esp32dev-weatherstation", + "esp32dev-gf-sun-inverter", + "esp32dev-ir", + "esp32dev-ble", + "esp32dev-ble-broker", + "esp32dev-ble-mqtt-undecoded", + "esp32dev-ble-aws", + "esp32dev-ble-datatest", + "esp32dev-ble-idf", + "esp32dev-rtl_433", + "esp32dev-rtl_433-fsk", + "esp32dev-multi_receiver", + "esp32dev-multi_receiver-pilight", + "esp32feather-ble", + "esp32-lolin32lite-ble", + "esp32-olimex-gtw-ble-eth", + "esp32-olimex-gtw-ble-poe", + "esp32-olimex-gtw-ble-poe-iso", + "esp32-wt32-eth01-ble-eth", + "esp32-olimex-gtw-ble-wifi", + "esp32-m5stick-ble", + "esp32-m5stack-ble", + "esp32-m5tough-ble", + "esp32-m5stick-c-ble", + "esp32-m5stick-cp-ble", + "esp32-m5atom-matrix", + "esp32-m5atom-lite", + "esp32doitv1-aithinker-r01-sx1278", + "esp32s3-dev-c1-ble", + "esp32s3-m5stack-stamps3", + "esp32s3-atomS3U", + "esp32c3-dev-m1-ble", + "esp32c3-dev-c2-ble", + "esp32c3-dev-c2-ble-no-serial", + "esp32c3_lolin_mini", + "esp32c3_lolin_mini_with_serial", + "esp32c3-m5stamp", + "esp32c3u-m5stamp", + "airm2m_core_esp32c3", + "heltec-rtl_433", + "heltec-rtl_433-fsk", + "heltec-ble", + "heltec-wifi-lora-32", + "lilygo-rtl_433", + "lilygo-rtl_433-fsk", + "lilygo-t3-s3-rtl_433", + "lilygo-t3-s3-rtl_433-fsk", + "lilygo-ble", + "tinypico-ble", + "ttgo-lora32-v1", + "ttgo-lora32-v21", + "ttgo-t-beam", + "thingpulse-espgateway", + "shelly-plus1", + "nodemcuv2-all-test", + "nodemcuv2-fastled-test", + "nodemcuv2-2g", + "nodemcuv2-ir", + "nodemcuv2-serial", + "nodemcuv2-rf", + "nodemcuv2-rf-cc1101", + "nodemcuv2-somfy-cc1101", + "nodemcuv2-rf2", + "nodemcuv2-rf2-cc1101", + "nodemcuv2-pilight", + "nodemcuv2-weatherstation", + "avatto-bakeey-ir", + "manual-wifi-test", + "rf-wifi-gateway", + "sonoff-basic", + "sonoff-basic-rfr3" + ], + "metadata": { + "totalCount": 83, + "categories": { + "esp32": 42, + "esp8266": 17, + "specialized": 24 + }, + "notes": [ + "This unified list contains all environments from both build.yml and build_and_docs_to_dev.yml", + "Previously build.yml had 85 environments and build_and_docs_to_dev.yml had 88", + "After deduplication and proper ordering, the total is 83 unique environments", + "Environments ending in -test or -all-test are typically excluded from production releases" + ] + } + } +} \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 08a0eeca..7c2b2125 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4,13 +4,9 @@ on: [push, pull_request] jobs: lint: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Check main format - uses: DoozyX/clang-format-lint-action@v0.6 - with: - source: "./main" - extensions: "h,ino" - clangFormatVersion: 9 + name: Lint code format + uses: ./.github/workflows/task-lint.yml + with: + source: 'main' + extensions: 'h,ino' + clang-format-version: '9' diff --git a/.github/workflows/manual_docs.yml b/.github/workflows/manual_docs.yml index 343a1d5b..e75414cf 100644 --- a/.github/workflows/manual_docs.yml +++ b/.github/workflows/manual_docs.yml @@ -1,45 +1,21 @@ name: Create and publish documentation + on: workflow_dispatch: workflow_call: + jobs: documentation: - runs-on: ubuntu-latest - name: Create the documentation and deploy it to GitHub Pages - steps: - - uses: actions/checkout@v4 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: "14.x" - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Install build dependencies - run: | - python -m pip install --upgrade pip - pip install requests pandas markdown pytablereader tabulate - npm install - - name: Download Common Config - run: | - curl -o docs/.vuepress/public/commonConfig.js https://www.theengs.io/commonConfig.js - - name: get lastest release tag - id: last_release - uses: InsonusK/get-latest-release@v1.0.1 - with: - myToken: ${{ github.token }} - view_top: 1 - - name: Set version tag from git - run: sed -i "s/version_tag/${{steps.last_release.outputs.tag_name}}/g" docs/.vuepress/config.js scripts/latest_version.json - - name: Build documentation - run: | - python ./scripts/generate_board_docs.py - python ./scripts/gen_wu.py ${GITHUB_REPOSITORY} - npm run docs:build - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: ./docs/.vuepress/dist - cname: docs.openmqttgateway.com \ No newline at end of file + name: Build and deploy production documentation + uses: ./.github/workflows/task-docs.yml + with: + python-version: '3.11' + node-version: '14.x' + version-source: 'release' + base-path: '/' + destination-dir: '.' + generate-webuploader: true + webuploader-args: '${GITHUB_REPOSITORY}' + run-pagespeed: false + secrets: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 31b2d31a..ee93156d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,47 +5,107 @@ on: types: [published] jobs: - build-upload: + prepare: runs-on: ubuntu-latest - name: Build and upload Assets to Release + outputs: + version-tag: ${{ steps.extract-tag.outputs.version }} + release-id: ${{ steps.extract-release.outputs.id }} + upload-url: ${{ steps.extract-release.outputs.upload_url }} + steps: + - name: Extract version tag + id: extract-tag + run: | + VERSION_TAG=${GITHUB_REF#refs/tags/} + echo "version=${VERSION_TAG}" >> $GITHUB_OUTPUT + echo "Extracted version: ${VERSION_TAG}" + + - name: Extract release info + id: extract-release + run: | + RELEASE_ID=$(jq --raw-output '.release.id' $GITHUB_EVENT_PATH) + UPLOAD_URL="https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}/assets{?name,label}" + echo "id=${RELEASE_ID}" >> $GITHUB_OUTPUT + echo "upload_url=${UPLOAD_URL}" >> $GITHUB_OUTPUT + echo "Release ID: ${RELEASE_ID}" + + build: + needs: prepare + name: Build release firmware + uses: ./.github/workflows/task-build.yml + with: + python-version: '3.13' + enable-dev-ota: false + version-tag: ${{ needs.prepare.outputs.version-tag }} + artifact-retention-days: 90 + artifact-name-prefix: 'firmware-' + prepare-for-deploy: true + + deploy: + needs: [prepare, build] + runs-on: ubuntu-latest + name: Deploy release assets steps: - uses: actions/checkout@v4 + + - name: Download all firmware artifacts + uses: actions/download-artifact@v4 + with: + pattern: firmware-* + path: .pio/build + merge-multiple: false + - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.13" + - name: Install uv uses: astral-sh/setup-uv@v6 with: version: "latest" enable-cache: false + - name: Install platformio run: | uv pip install --system -U https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip - - name: Set version tag from git - run: sed -i "s/version_tag/${GITHUB_REF#refs/tags/}/g" main/User_config.h scripts/latest_version.json - - name: Extract ESP32 platform version from platformio.ini + + - name: Reorganize artifacts for prepare_deploy.sh run: | - ESP32_VERSION=$(grep 'esp32_platform\s*=' platformio.ini | cut -d'@' -f2 | tr -d '[:space:]') - echo "ESP32_PLATFORM_VERSION=${ESP32_VERSION}" >> $GITHUB_ENV - - name: Run PlatformIO - run: platformio run + # Move artifacts from download structure to expected .pio/build structure + for env_dir in .pio/build/firmware-*; do + if [ -d "$env_dir" ]; then + env_name=$(basename "$env_dir" | sed 's/^firmware-//') + mkdir -p ".pio/build/${env_name}" + mv "$env_dir"/*.bin ".pio/build/${env_name}/" 2>/dev/null || true + rm -rf "$env_dir" + fi + done + - name: Prepare Release Assets run: | sudo apt install rename ./scripts/prepare_deploy.sh - - name: Get upload url - id: release-id - run: | - RELEASE_ID=$(jq --raw-output '.release.id' $GITHUB_EVENT_PATH) - echo "::set-output name=upload_url::https://uploads.github.com/repos/${GITHUB_REPOSITORY}/releases/${RELEASE_ID}/assets{?name,label}" + - name: Upload Release Assets uses: bgpat/release-asset-action@03b0c30db1c4031ce3474740b0e4275cd7e126a3 with: pattern: "toDeploy/*" github-token: ${{ secrets.GITHUB_TOKEN }} - release-url: ${{ steps.release-id.outputs.upload_url }} + release-url: ${{ needs.prepare.outputs.upload-url }} allow-overwrite: true - call-workflow-passing-data: - needs: build-upload - uses: ./.github/workflows/manual_docs.yml + + documentation: + needs: [prepare, deploy] + name: Build and deploy release documentation + uses: ./.github/workflows/task-docs.yml + with: + python-version: '3.11' + node-version: '18.x' + version-source: 'git-tag' + base-path: '/' + destination-dir: '.' + generate-webuploader: true + run-pagespeed: false + secrets: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + APIKEY: ${{ secrets.APIKEY }} diff --git a/.github/workflows/task-build.yml b/.github/workflows/task-build.yml new file mode 100644 index 00000000..787ed1f3 --- /dev/null +++ b/.github/workflows/task-build.yml @@ -0,0 +1,104 @@ +name: Reusable Build Workflow + +on: + workflow_call: + inputs: + python-version: + description: 'Python version to use' + required: false + type: string + default: '3.13' + enable-dev-ota: + description: 'Enable development OTA builds' + required: false + type: boolean + default: false + version-tag: + description: 'Version tag to inject into firmware (e.g., SHA or release tag)' + required: false + type: string + default: 'unspecified' + artifact-retention-days: + description: 'Number of days to retain build artifacts' + required: false + type: number + default: 7 + artifact-name-prefix: + description: 'Prefix for artifact names (e.g., "firmware-" for dev builds)' + required: false + type: string + default: '' + prepare-for-deploy: + description: 'Prepare firmware files for deployment with specific naming' + required: false + type: boolean + default: false + +jobs: + load-environments: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + - id: set-matrix + run: | + ENVIRONMENTS=$(jq -c '.environments.all' .github/workflows/environments.json) + echo "matrix=${ENVIRONMENTS}" >> $GITHUB_OUTPUT + + build: + needs: load-environments + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + environments: ${{ fromJson(needs.load-environments.outputs.matrix) }} + name: Build ${{ matrix.environments }} + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + version: "latest" + enable-cache: false + + - name: Install PlatformIO dependencies + run: | + uv pip install --system -U https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip + + - name: Build firmware using ci.sh + run: | + BUILD_ARGS="${{ matrix.environments }}" + + # Add version tag if specified + if [ "${{ inputs.version-tag }}" != "unspecified" ]; then + BUILD_ARGS="$BUILD_ARGS --version ${{ inputs.version-tag }}" + fi + + # Add mode flag (dev/prod) + if [ "${{ inputs.enable-dev-ota }}" = "true" ]; then + BUILD_ARGS="$BUILD_ARGS --mode dev" + else + BUILD_ARGS="$BUILD_ARGS --mode prod" + fi + + # Add deploy flag if needed + if [ "${{ inputs.prepare-for-deploy }}" = "true" ]; then + BUILD_ARGS="$BUILD_ARGS --deploy-ready" + fi + + # Execute build (uses default generated/artifacts/ directory) + ./scripts/ci.sh build $BUILD_ARGS + + - name: Upload firmware artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.artifact-name-prefix }}${{ matrix.environments }} + path: generated/artifacts/ + retention-days: ${{ inputs.artifact-retention-days }} diff --git a/.github/workflows/task-docs.yml b/.github/workflows/task-docs.yml new file mode 100644 index 00000000..6f962518 --- /dev/null +++ b/.github/workflows/task-docs.yml @@ -0,0 +1,127 @@ +name: Reusable Documentation Workflow + +on: + workflow_call: + inputs: + python-version: + description: 'Python version to use' + required: false + type: string + default: '3.11' + node-version: + description: 'Node.js version to use' + required: false + type: string + default: '14.x' + version-source: + description: 'Source of version: "release" (from GitHub release) or "custom" (provided)' + required: false + type: string + default: 'release' + custom-version: + description: 'Custom version string when version-source is "custom"' + required: false + type: string + default: '' + base-path: + description: 'Base path for documentation (e.g., "/" for prod, "/dev/" for dev)' + required: false + type: string + default: '/' + destination-dir: + description: 'Destination directory for GitHub Pages deployment' + required: false + type: string + default: '.' + generate-webuploader: + description: 'Generate WebUploader manifest' + required: false + type: boolean + default: true + webuploader-args: + description: 'Additional arguments for gen_wu.py script' + required: false + type: string + default: '' + run-pagespeed: + description: 'Run PageSpeed Insights after deployment' + required: false + type: boolean + default: false + pagespeed-url: + description: 'URL to test with PageSpeed Insights' + required: false + type: string + default: 'https://docs.openmqttgateway.com/' + secrets: + GITHUB_TOKEN: + required: true + APIKEY: + required: false + +jobs: + documentation: + runs-on: ubuntu-latest + name: Create and deploy documentation + steps: + - uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Build documentation site + run: | + # Build script arguments based on inputs + ARGS="" + + # Set version source and custom version + if [ "${{ inputs.version-source }}" = "custom" ] && [ -n "${{ inputs.custom-version }}" ]; then + ARGS="$ARGS --custom-version '${{ inputs.custom-version }}' --version-source custom" + elif [ "${{ inputs.version-source }}" = "release" ]; then + # Get latest release tag + if command -v git >/dev/null 2>&1; then + LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "development") + ARGS="$ARGS --custom-version '${LATEST_TAG}' --version-source release" + else + ARGS="$ARGS --version-source release" + fi + fi + + # Set base path/URL prefix + if [ "${{ inputs.base-path }}" != "/" ]; then + ARGS="$ARGS --url-prefix '${{ inputs.base-path }}'" + fi + + # WebUploader generation + if [ "${{ inputs.generate-webuploader }}" != "true" ]; then + ARGS="$ARGS --no-webuploader" + elif [ -n "${{ inputs.webuploader-args }}" ]; then + ARGS="$ARGS --webuploader-args '${{ inputs.webuploader-args }}'" + fi + + # Run the build script + eval "./scripts/ci_site.sh ${ARGS}" + + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./generated/site + destination_dir: ${{ inputs.destination-dir }} + cname: docs.openmqttgateway.com + + - name: Run PageSpeed Insights + if: inputs.run-pagespeed + uses: jakepartusch/psi-action@v1.3 + id: psi + with: + url: ${{ inputs.pagespeed-url }} + threshold: 60 + key: ${{ secrets.APIKEY }} diff --git a/.github/workflows/task-lint.yml b/.github/workflows/task-lint.yml new file mode 100644 index 00000000..e7ba6acf --- /dev/null +++ b/.github/workflows/task-lint.yml @@ -0,0 +1,40 @@ +name: Reusable Lint Workflow + +on: + workflow_call: + inputs: + source: + description: 'Source directory to lint (single directory)' + required: false + type: string + default: 'main' + extensions: + description: 'File extensions to check (comma-separated)' + required: false + type: string + default: 'h,ino,cpp' + clang-format-version: + description: 'clang-format version to use' + required: false + type: string + default: '9' + +jobs: + lint: + runs-on: ubuntu-latest + name: Check code format + steps: + - uses: actions/checkout@v4 + + - name: Install clang-format + run: | + sudo apt-get update + sudo apt-get install -y clang-format + + - name: Check code format with ci.sh qa + run: | + ./scripts/ci.sh qa \ + --check \ + --source "${{ inputs.source }}" \ + --extensions "${{ inputs.extensions }}" \ + --clang-format-version "${{ inputs.clang-format-version }}" \ No newline at end of file diff --git a/.gitignore b/.gitignore index 84ef10d0..18c8a66c 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,15 @@ managed_components .github/chatmodes .github/prompts -.github/*instructions.md \ No newline at end of file +.github/*instructions.md +.github/workflows/_docs +# CI/CD Generated outputs +docs/.vuepress/dist/ +docs/.vuepress/components/ +docs/.vuepress/public/ +^docs/.vuepress/public/img/ +^docs/.vuepress/public/*.png +^docs/.vuepress/public/*.txt + +generated/ + diff --git a/scripts/CI_SCRIPTS.md b/scripts/CI_SCRIPTS.md new file mode 100644 index 00000000..e93d37e8 --- /dev/null +++ b/scripts/CI_SCRIPTS.md @@ -0,0 +1,575 @@ +# CI/CD Scripts Documentation + +This documentation describes the CI/CD scripts used to build OpenMQTTGateway firmware and documentation. These scripts work in GitHub Actions, locally, and in any CI/CD environment. + +- [Overview](#cicd-scripts-documentation) +- [Quick Reference](#quick-reference) + - [Script Hierarchy](#script-hierarchy) + - [Script Description](#script-description) + - [Output Structure](#output-structure) +- [Commands](#commands) + - [ci.sh - Main Entry Point](#commands) + - [ci.sh build - Build Firmware](#cish-build---build-firmware) + - [ci.sh site - Build Documentation](#cish-site---build-documentation) + - [ci.sh qa - Code Formatting Check](#cish-qa---code-formatting-check) +- [Internal Scripts](#internal-scripts) + - [ci_set_version.sh](#ci_set_versionsh) + - [ci_build_firmware.sh](#ci_build_firmwaresh) + - [ci_prepare_artifacts.sh](#ci_prepare_artifactssh) + - [ci_00_config.sh](#ci_00_configsh) +- [Python Helper Scripts](#python-helper-scripts) + - [generate_board_docs.py](#generate_board_docspy) + - [gen_wu.py](#gen_wupy) +- [Environment Variables](#environment-variables) +- [Exit Codes](#exit-codes) +- [Environment Detection](#environment-detection) + + + +## Quick Reference + +### Script Hierarchy + +``` +ci.sh (dispatcher) +├── build → ci_build.sh → ci_set_version.sh +│ → ci_build_firmware.sh +│ → ci_prepare_artifacts.sh +├── site → ci_site.sh → generate_board_docs.py +│ → gen_wu.py +├── qa → ci_qa.sh → clang-format +└── all → qa + build + site (sequential) +``` + +### Script Description + +| Script | Purpose | Called By | +|--------|---------|-----------| +| `ci.sh` | Main command dispatcher | User/GitHub Actions | +| `ci_build.sh` | Build firmware orchestrator | ci.sh build | +| `ci_site.sh` | Documentation build orchestrator | ci.sh site | +| `ci_qa.sh` | Code formatting checker | ci.sh qa | +| `ci_set_version.sh` | Version injection in firmware | ci_build.sh | +| `ci_build_firmware.sh` | PlatformIO build execution | ci_build.sh | +| `ci_prepare_artifacts.sh` | Artifact packaging | ci_build.sh | +| `ci_00_config.sh` | Shared configuration and functions | All scripts | + + +### Output Structure + +Build outputs are organized in the project root: + +``` +.pio/build// # PlatformIO build outputs +├── firmware.bin # Main firmware binary +├── bootloader.bin # ESP32 bootloader +└── partitions.bin # ESP32 partition table + +generated/ +├── artifacts/ # Packaged firmware artifacts +└── site/ # Built documentation (VuePress output) + +scripts/ +├── latest_version.json # Production version metadata +└── latest_version_dev.json # Development version metadata +``` + +## Commands + +`ci.sh` is the main Entry Point. Command dispatcher that routes to specialized scripts. + +**Usage:** +```bash +./scripts/ci.sh [OPTIONS] +``` + +**Commands:** +- `build` - Build firmware for specified environment +- `site` or `docs` - Build documentation website +- `qa` or `lint` - Run code formatting checks +- `all` or `pipeline` - Run complete pipeline (qa + build + site) + +**Examples:** +```bash +# Get Help +./scripts/ci.sh build --help +./scripts/ci.sh qa --help +./scripts/ci.sh site --help + +# Build firmware +./scripts/ci.sh build esp32dev-ble --mode dev +./scripts/ci.sh build esp32dev-all-test --version v1.8.0 --deploy-ready + +# Build documentation +./scripts/ci.sh site --mode prod +./scripts/ci.sh site --mode dev --url-prefix /dev/ + +# Check code formatting +./scripts/ci.sh qa --check +./scripts/ci.sh qa --fix + +# Run complete pipeline +./scripts/ci.sh all esp32dev-ble --version v1.8.0 +./scripts/ci.sh all esp32dev-ble --no-site +``` + +**Options for `all` command:** +- `--no-site` - Skip documentation build step +- All options from `build` command are passed through + +--- + +### ci.sh build - Build Firmware + +Orchestrates complete firmware build: version injection, compilation, artifact packaging. + +**Usage:** +```bash +./scripts/ci.sh build [OPTIONS] +``` + +**Required Arguments:** +- `` - PlatformIO environment name (e.g., esp32dev-ble, nodemcuv2-rf) + +**Options:** +- `--version ` - Version string to inject (default: auto-generated from git) +- `--mode ` - Build mode (default: prod) + - `dev` - Enables development OTA, sets DEVELOPMENTOTA=true + - `prod` - Standard production build +- `--deploy-ready` - Package artifacts with deployment naming (env-firmware.bin) +- `--output ` - Output directory for artifacts (default: generated/artifacts) +- `--skip-verification` - Skip build tools verification +- `--clean` - Clean previous build before starting +- `--verbose` - Enable verbose PlatformIO output +- `--list-envs` - List all available PlatformIO environments +- `--help` - Show help message + +**Execution Flow:** +``` +ci.sh build esp32dev-ble --version v1.8.0 --mode prod --deploy-ready + │ + ├─> ci_build.sh (orchestrator) + │ ├─> verify_build_tools() - Check python3, platformio, git + │ ├─> ci_set_version.sh v1.8.0 - Inject version in User_config.h + │ ├─> ci_build_firmware.sh esp32dev-ble - Execute PlatformIO build + │ └─> ci_prepare_artifacts.sh esp32dev-ble --deploy - Package binaries + │ + └─> Outputs in generated/artifacts/ + ├─ esp32dev-ble-firmware.bin + ├─ esp32dev-ble-bootloader.bin + └─ esp32dev-ble-partitions.bin +``` + +**Examples:** +```bash +# Development build +./scripts/ci.sh build esp32dev-ble --mode dev + +# Production build with version +./scripts/ci.sh build esp32dev-ble --version v1.8.0 --mode prod + +# Deploy-ready build +./scripts/ci.sh build esp32dev-all-test --version v1.8.0 --deploy-ready + +# Clean build with verbose output +./scripts/ci.sh build nodemcuv2-rf --clean --verbose + +# List available environments +./scripts/ci.sh build --list-envs +``` + +**Environment Variables:** +- `CI` - Set to 'true' in CI/CD environments +- `BUILD_NUMBER` - Build number from CI/CD system +- `GIT_COMMIT` - Git commit hash for auto-versioning +- `PLATFORMIO_BUILD_FLAGS` - Additional PlatformIO flags (set by script when --mode dev) + +**Output Files:** +- Standard mode: `firmware.bin`, `partitions.bin` in generated/artifacts/ +- Deploy mode: `-firmware.bin`, `-bootloader.bin`, `-partitions.bin` + +--- + +### ci.sh site - Build Documentation + +Builds VuePress documentation website with version management and WebUploader manifest generation. + +**Usage:** +```bash +./scripts/ci.sh site [OPTIONS] +``` + +**Options:** +- `--mode ` - Documentation mode (default: prod) + - `dev` - Development documentation with watermark + - `prod` - Production documentation +- `--version-source ` - Version source (default: release) + - `release` - Use git tag as version + - `custom` - Use custom version string +- `--custom-version ` - Custom version string (requires --version-source custom) +- `--url-prefix ` - Base URL path (default: /) + - Example: `/dev/` for development subdirectory +- `--no-webuploader` - Skip WebUploader manifest generation +- `--webuploader-args ` - Additional arguments for gen_wu.py +- `--preview` - Open browser after build (local development) +- `--help` - Show help message + +**Execution Flow:** +``` +ci.sh site --mode prod --version-source release + │ + ├─> ci_site.sh (orchestrator) + │ ├─> check_requirements() - Verify node, npm, python3, pip3 + │ ├─> install_dependencies() - npm install, pip3 install packages + │ ├─> download_common_config() - Fetch from theengs.io + │ ├─> get_version() - Extract from git tag or use custom + │ ├─> set_version() - Update VuePress config and JSON files + │ ├─> set_url_prefix() - Set base path in config + │ ├─> generate_board_docs.py - Auto-generate board documentation + │ ├─> npm run docs:build - Build VuePress site + │ └─> gen_wu.py - Generate WebUploader manifest + │ + └─> Outputs in generated/site/ + ├─ index.html + ├─ assets/ + └─ [board documentation pages] +``` + +**Examples:** +```bash +# Production documentation +./scripts/ci.sh site --mode prod + +# Development documentation with custom version +./scripts/ci.sh site --mode dev --version-source custom --custom-version "DEVELOPMENT SHA:abc123" + +# Documentation for /dev/ subdirectory +./scripts/ci.sh site --mode dev --url-prefix /dev/ + +# Skip WebUploader manifest +./scripts/ci.sh site --no-webuploader + +# Local preview +./scripts/ci.sh site --preview +``` + +**Required Tools:** +- Node.js (for VuePress) +- npm (for package management) +- Python 3 (for board docs generator) +- pip3 (for Python dependencies: requests, pandas, markdown, pytablereader, tabulate) + +**Output Files:** +- `generated/site/` - Complete static website +- `scripts/latest_version.json` - Production version metadata (updated) +- `scripts/latest_version_dev.json` - Development version metadata (updated) + +--- + +### ci.sh qa - Code Formatting Check + +Checks and fixes code formatting using clang-format. + +**Usage:** +```bash +./scripts/ci.sh qa [OPTIONS] +``` + +**Options:** +- `--check` - Check formatting without modifying files (default) +- `--fix` - Automatically fix formatting issues +- `--source ` - Source directory to check (default: main) +- `--extensions ` - File extensions to check, comma-separated (default: h,ino,cpp) +- `--clang-format-version ` - clang-format version to use (default: 9) +- `--verbose` - Show detailed output for each file +- `--help` - Show help message + +**Execution Flow:** +``` +ci.sh qa --check --source main --extensions h,ino + │ + ├─> ci_qa.sh (orchestrator) + │ ├─> check_clang_format() - Find clang-format-9 or clang-format + │ ├─> find_files() - Locate files matching extensions in source dir + │ └─> check_formatting() - Run clang-format --dry-run --Werror + │ └─> Report files with formatting issues + │ + └─> Exit code: 0 (pass) or 1 (formatting issues found) +``` + +**Examples:** +```bash +# Check formatting (CI mode) +./scripts/ci.sh qa --check + +# Fix formatting automatically +./scripts/ci.sh qa --fix + +# Check specific directory +./scripts/ci.sh qa --check --source lib + +# Check only .h and .ino files +./scripts/ci.sh qa --check --extensions h,ino + +# Check with verbose output +./scripts/ci.sh qa --check --verbose + +# Use different clang-format version +./scripts/ci.sh qa --check --clang-format-version 11 +``` + +**Required Tools:** +- clang-format (version specified, default: 9) + - Install: `sudo apt-get install clang-format-9` + +**Output:** +- Check mode: Lists files with formatting issues and shows diffs +- Fix mode: Modifies files in-place and reports changes +- Exit code 0: All files properly formatted +- Exit code 1: Formatting issues found (in check mode) + +--- + +## Internal Scripts + +These scripts are called by the main orchestrators. It can be invoked directly but is not raccomanded. + +### ci_set_version.sh + +Injects version string into firmware configuration files. + +**Called By:** `ci_build.sh` + +**Usage:** +```bash +./scripts/ci_set_version.sh [--dev] +``` + +**Arguments:** +- `` - Version string to inject (e.g., v1.8.0 or abc123) +- `--dev` - Development mode (updates latest_version_dev.json) + +**Files Modified:** +- `main/User_config.h` - Replaces "version_tag" with actual version +- `scripts/latest_version.json` - Production version metadata +- `scripts/latest_version_dev.json` - Development version metadata (--dev mode) + +**Behavior:** +- Creates .bak backup files before modification +- Replaces all occurrences of "version_tag" string +- Validates version string (must not be empty or "version_tag") +- Cleans up backup files on success + +--- + +### ci_build_firmware.sh + +Executes PlatformIO build for specified environment. + +**Called By:** `ci_build.sh` + +**Usage:** +```bash +./scripts/ci_build_firmware.sh [OPTIONS] +``` + +**Arguments:** +- `` - PlatformIO environment name + +**Options:** +- `--dev-ota` - Enable development OTA (sets PLATFORMIO_BUILD_FLAGS) +- `--clean` - Clean before build +- `--verbose` - Verbose PlatformIO output + +**Environment Variables Set:** +- `PYTHONIOENCODING=utf-8` +- `PYTHONUTF8=1` +- `PLATFORMIO_BUILD_FLAGS="-DDEVELOPMENTOTA=true"` (when --dev-ota) + +**PlatformIO Command:** +```bash +platformio run -e [--verbose] +``` + +**Output Location:** +- `.pio/build//firmware.bin` +- `.pio/build//bootloader.bin` (ESP32 only) +- `.pio/build//partitions.bin` (ESP32 only) + +--- + +### ci_prepare_artifacts.sh + +Packages firmware binaries from PlatformIO build directory. + +**Called By:** `ci_build.sh` + +**Usage:** +```bash +./scripts/ci_prepare_artifacts.sh [OPTIONS] +``` + +**Arguments:** +- `` - PlatformIO environment name + +**Options:** +- `--deploy` - Use deployment naming (prefix with environment name) +- `--output ` - Output directory (default: generated/artifacts) + +**Behavior:** + +Standard mode (no --deploy): +- Copies: `firmware.bin`, `partitions.bin` +- Does NOT copy: `bootloader.bin` + +Deploy mode (with --deploy): +- Copies and renames: + - `firmware.bin` → `-firmware.bin` + - `bootloader.bin` → `-bootloader.bin` + - `partitions.bin` → `-partitions.bin` + +**Source Location:** +- `.pio/build//` + +**Output Location:** +- Specified by `--output` or default `generated/artifacts/` + +--- + +### ci_00_config.sh + +Shared configuration and helper functions for all CI scripts. + +**Sourced By:** All ci_*.sh scripts + +**Provides:** +- Color codes for terminal output (BLUE, GREEN, RED, YELLOW, NC) +- Logging functions: `log_info()`, `log_warn()`, `log_error()`, `log_success()` +- Path constants: `BUILD_DIR`, `ARTIFACTS_DIR`, `SITE_DIR` +- Common utility functions + +**Constants Defined:** +- `BUILD_DIR=".pio/build"` - PlatformIO build directory +- `ARTIFACTS_DIR="generated/artifacts"` - Artifact output directory +- `SITE_DIR="generated/site"` - Documentation output directory + +**Logging Functions:** +```bash +log_info "message" # Blue [INFO] prefix +log_warn "message" # Yellow [WARN] prefix +log_error "message" # Red [ERROR] prefix +log_success "message" # Green [SUCCESS] prefix +``` + +--- + +## Python Helper Scripts + +Other scripts are present and used as internal scripts and it's used as retrocompatibility. Below the lists: + - `generate_board_docs.py` + - `gen_wu.py` + +### generate_board_docs.py + +Auto-generates board-specific documentation pages from platformio.ini. + +**Called By:** `ci_site.sh` + +**Usage:** +```bash +python3 ./scripts/generate_board_docs.py +``` + +**Input:** +- `platformio.ini` - Board configurations +- `environments.ini` - Additional environments + +**Output:** +- Markdown files in `docs/` directory for each board configuration + +**Purpose:** +- Creates documentation pages for each hardware board +- Extracts configuration details from PlatformIO environment definitions +- Formats technical specifications and pin mappings + +--- + +### gen_wu.py + +Generates WebUploader manifest for OTA firmware updates. + +**Called By:** `ci_site.sh` + +**Usage:** +```bash +python3 ./scripts/gen_wu.py [--dev] [repository] +``` + +**Arguments:** +- `--dev` - Generate development manifest +- `repository` - GitHub repository name (e.g., 1technophile/OpenMQTTGateway) + +**Input:** +- `.pio/build//firmware.bin` - Compiled firmware files +- `scripts/latest_version.json` or `scripts/latest_version_dev.json` + +**Output:** +- WebUploader manifest JSON file in `docs/.vuepress/public/` + +**Purpose:** +- Creates manifest for web-based firmware updater +- Lists available firmware files with metadata +- Used by documentation site for OTA updates + +--- + +## Environment Variables + +Scripts respect these environment variables: + +- `PYTHONIOENCODING=utf-8`: Python encoding +- `PYTHONUTF8=1`: UTF-8 mode +- `PLATFORMIO_BUILD_FLAGS`: Custom build flags +- `ESP32_PLATFORM_VERSION`: Extracted automatically + +--- + +## Exit Codes + +All scripts use standard exit codes: + +- `0` - Success +- `1` - General error or failure +- `2` - Missing required tools or dependencies + +Scripts use `set -euo pipefail` for strict error handling: +- `-e` - Exit on error +- `-u` - Exit on undefined variable +- `-o pipefail` - Exit on pipe failure + +--- + +## Environment Detection + +Scripts automatically detect if running in CI/CD: + +```bash +if [[ "${CI:-false}" == "true" ]]; then + # Running in CI/CD + # Disable interactive prompts + # Use different output formatting +fi +``` + +CI/CD environments typically set: +- `CI=true` +- `GITHUB_ACTIONS=true` (GitHub Actions) +- `BUILD_NUMBER` (build number) +- `GIT_COMMIT` (commit hash) + + +--- + +This documentation reflects the current implementation of CI/CD scripts. All scripts are located in `./scripts/` directory. + +For GitHub Actions workflow documentation, see `.github/workflows/README.md`. diff --git a/scripts/add_c_flags.py b/scripts/add_c_flags.py index 902a95d4..f9a7f216 100644 --- a/scripts/add_c_flags.py +++ b/scripts/add_c_flags.py @@ -1,3 +1,5 @@ +# Adds compiler flags to suppress warnings during PlatformIO build +# Used by: PlatformIO environments (esp32dev-pilight*, esp32-m5stick-c*) Import("env") diff --git a/scripts/ci.sh b/scripts/ci.sh new file mode 100755 index 00000000..6f68783e --- /dev/null +++ b/scripts/ci.sh @@ -0,0 +1,174 @@ +#!/bin/bash +# CI/CD Main Entry Point - Command Dispatcher +# Routes commands to specialized scripts for build, site, qa, and all +# Usage: ./scripts/ci.sh [OPTIONS] + +set -euo pipefail + +# Constants +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +# Load shared configuration (colors, logging functions, paths) +if [[ -f "${SCRIPT_DIR}/ci_00_config.sh" ]]; then + source "${SCRIPT_DIR}/ci_00_config.sh" +else + echo "ERROR: ci_00_config.sh not found" >&2 + exit 1 +fi + +# Function to print banner +print_banner() { + echo "╔════════════════════════════════════════╗" + echo "║ OpenMQTTGateway CI/CD Pipeline ║" + echo "╚════════════════════════════════════════╝" + echo "" +} + +# Show usage +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +OpenMQTTGateway CI/CD Pipeline - Main Entry Point + +Commands: + build Build firmware for specified environment + site Build and deploy documentation/website + qa Run quality assurance checks (linting, formatting) + all Run complete pipeline (qa + build + site) + +Examples: + # Build firmware + $0 build esp32dev-all-test --mode dev + $0 build esp32dev-bt --version v1.8.0 --deploy-ready + + # Build and deploy documentation + $0 site --mode prod --deploy + $0 site --mode dev --preview + + # Run quality checks + $0 qa --check + $0 qa --fix + + # Run complete pipeline + $0 all esp32dev-bt --version v1.8.0 + +Get help for specific commands: + $0 build --help + $0 site --help + $0 qa --help + +EOF + exit 0 +} + +# Function to run build pipeline +run_build_pipeline() { + log_info "Executing build pipeline..." + "${SCRIPT_DIR}/ci_build.sh" "$@" +} + +# Function to run site pipeline +run_site_pipeline() { + log_info "Executing site pipeline..." + "${SCRIPT_DIR}/ci_site.sh" "$@" +} + +# Function to run QA pipeline +run_qa_pipeline() { + log_info "Executing QA pipeline..." + "${SCRIPT_DIR}/ci_qa.sh" "$@" +} + +# Function to run complete pipeline +run_all_pipeline() { + local start_time + start_time=$(date +%s) + + log_info "Starting complete CI/CD pipeline..." + echo "" + + # Step 1: Quality Assurance + log_info "═══ Step 1/3: Quality Assurance ═══" + run_qa_pipeline --check || { + log_error "QA checks failed. Pipeline aborted." + return 1 + } + echo "" + + # Step 2: Build Firmware + log_info "═══ Step 2/3: Build Firmware ═══" + run_build_pipeline "$@" || { + log_error "Build failed. Pipeline aborted." + return 1 + } + echo "" + + # Step 3: Build Site (only if not in --no-site mode) + if [[ ! " $* " =~ " --no-site " ]]; then + log_info "═══ Step 3/3: Build Documentation ═══" + run_site_pipeline --mode prod || { + log_warn "Site build failed, but continuing..." + } + else + log_info "Skipping site build (--no-site flag)" + fi + + local end_time + end_time=$(date +%s) + local duration=$((end_time - start_time)) + + echo "" + echo "╔════════════════════════════════════════╗" + echo "║ Complete Pipeline Summary ║" + echo "╚════════════════════════════════════════╝" + echo " Total Duration: ${duration}s" + echo " Status: SUCCESS ✓" + echo "╚════════════════════════════════════════╝" +} + +# Main execution +main() { + # Check if no arguments provided + if [[ $# -eq 0 ]]; then + print_banner + usage + fi + + # Get command + local command="$1" + shift || true + + # Handle help flags + if [[ "$command" == "--help" || "$command" == "-h" ]]; then + print_banner + usage + fi + + print_banner + + # Route to appropriate pipeline + case "$command" in + build) + run_build_pipeline "$@" + ;; + site|docs) + run_site_pipeline "$@" + ;; + qa|lint) + run_qa_pipeline "$@" + ;; + all|pipeline) + run_all_pipeline "$@" + ;; + *) + log_error "Unknown command: $command" + echo "" + usage + ;; + esac +} + +# Execute main function +main "$@" diff --git a/scripts/ci_00_config.sh b/scripts/ci_00_config.sh new file mode 100755 index 00000000..bf9b8d8a --- /dev/null +++ b/scripts/ci_00_config.sh @@ -0,0 +1,58 @@ +# Build Scripts Configuration +# Used by: All build scripts for centralized configuration + +# Python Configuration +PYTHON_VERSION="3.13" +PLATFORMIO_VERSION="https://github.com/pioarduino/platformio-core/archive/refs/tags/v6.1.18.zip" + +# Centralized Output Directory Structure +# All CI/CD generated files go under generated/ +GENERATED_BASE_DIR="generated" +ARTIFACTS_DIR="${GENERATED_BASE_DIR}/artifacts" +SITE_DIR="${GENERATED_BASE_DIR}/site" +REPORTS_DIR="${GENERATED_BASE_DIR}/reports" + +# PlatformIO Directory Configuration +BUILD_DIR=".pio/build" +SCRIPTS_DIR="scripts" + +# Build Configuration +DEFAULT_ENVIRONMENT="esp32dev-all-test" +ENABLE_VERBOSE_BUILD="false" +ENABLE_BUILD_CACHE="true" + +# Artifact Configuration +ARTIFACT_RETENTION_DAYS="7" +CREATE_MANIFEST="true" +COMPRESS_ARTIFACTS="false" + +# Version Configuration +VERSION_FILE_PROD="scripts/latest_version.json" +VERSION_FILE_DEV="scripts/latest_version_dev.json" +USER_CONFIG_FILE="main/User_config.h" + +# Logging Configuration +ENABLE_COLOR_OUTPUT="true" +LOG_LEVEL="INFO" # DEBUG, INFO, WARN, ERROR + +# ============================================================================ +# Colors - ANSI color codes for terminal output +# ============================================================================ +readonly RED='\033[0;31m' +readonly GREEN='\033[0;32m' +readonly YELLOW='\033[1;33m' +readonly BLUE='\033[0;34m' +readonly NC='\033[0m' # No Color + +# ============================================================================ +# Logging Functions - Standardized logging across all build scripts +# ============================================================================ +log_info() { echo -e "${GREEN}[INFO]${NC} $*" >&2; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } +log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } +log_step() { echo -e "${BLUE}[STEP]${NC} $*" >&2; } + +# Advanced Options +ENABLE_CCACHE="false" +CCACHE_DIR=".ccache" +MAX_BUILD_JOBS="4" diff --git a/scripts/ci_build.sh b/scripts/ci_build.sh new file mode 100755 index 00000000..67bb33ba --- /dev/null +++ b/scripts/ci_build.sh @@ -0,0 +1,384 @@ +#!/bin/bash +# CI/CD agnostic wrapper for complete build pipeline +# Orchestrates all build scripts with a single command +# Usage: ./scripts/ci.sh [OPTIONS] + +set -euo pipefail + +# Constants +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +# Load shared configuration (colors, logging functions, paths) +if [[ -f "${SCRIPT_DIR}/ci_00_config.sh" ]]; then + source "${SCRIPT_DIR}/ci_00_config.sh" +else + echo "ERROR: ci_00_config.sh not found" >&2 + exit 1 +fi + +# Function to print banner +print_banner() { + echo "╔════════════════════════════════════════╗" + echo "║ OpenMQTTGateway CI/CD Build ║" + echo "╚════════════════════════════════════════╝" + echo "" +} + +# Function to print summary +print_summary() { + local env="$1" + local version="$2" + local start_time="$3" + local end_time + end_time=$(date +%s) + local duration=$((end_time - start_time)) + + echo "" + echo "╔════════════════════════════════════════╗" + echo "║ Build Summary ║" + echo "╚════════════════════════════════════════╝" + echo " Environment: $env" + echo " Version: $version" + echo " Duration: ${duration}s" + echo " Status: SUCCESS ✓" + echo "╚════════════════════════════════════════╝" +} + +# Function to check if command exists +command_exists() { + command -v "$1" >/dev/null 2>&1 +} + +# Function to get command version +get_command_version() { + local cmd="$1" + case "$cmd" in + platformio) + platformio --version 2>&1 | head -n1 | grep -oP '\d+\.\d+\.\d+' || echo "unknown" + ;; + python|python3) + python3 --version 2>&1 | grep -oP '\d+\.\d+' || echo "unknown" + ;; + *) + echo "unknown" + ;; + esac +} + +# Function to verify required tools +verify_build_tools() { + log_info "Verifying required build tools..." + + local missing_tools=() + local version_mismatch=() + + # Check Python + if ! command_exists python3; then + missing_tools+=("python3") + else + local python_version + python_version=$(get_command_version python3) + log_info "✓ Python ${python_version} found" + fi + + # Check PlatformIO + if ! command_exists platformio; then + missing_tools+=("platformio") + else + local pio_version + pio_version=$(get_command_version platformio) + log_info "✓ PlatformIO ${pio_version} found" + fi + + # Check git (for version auto-generation) + if ! command_exists git; then + log_warn "git not found (optional, but recommended)" + else + log_info "✓ git found" + fi + + # Report missing tools + if [[ ${#missing_tools[@]} -gt 0 ]]; then + log_error "Missing required tools: ${missing_tools[*]}" + log_error "" + log_error "Please install missing tools:" + for tool in "${missing_tools[@]}"; do + case "$tool" in + python3) + log_error " - Python 3: https://www.python.org/downloads/" + ;; + platformio) + log_error " - PlatformIO: pip3 install platformio" + log_error " or: pip3 install ${PLATFORMIO_VERSION:-platformio}" + ;; + esac + done + log_error "" + log_error "Or skip this check with: --skip-verification" + return 1 + fi + + log_info "All required tools are available" + return 0 +} + +# Function to cleanup on error +cleanup_on_error() { + log_error "Build failed, cleaning up..." + # Restore any backups + find . -name "*.bak" -type f -exec bash -c 'mv "$1" "${1%.bak}"' _ {} \; 2>/dev/null || true +} + +# Function to list available environments +list_environments() { + log_info "Available PlatformIO environments:" + echo "" + + local env_files=("${PROJECT_ROOT}/platformio.ini" "${PROJECT_ROOT}/environments.ini") + local envs=() + + for file in "${env_files[@]}"; do + if [[ -f "$file" ]]; then + while IFS= read -r line; do + if [[ "$line" =~ ^\[env:([^\]]+)\] ]]; then + local env_name="${BASH_REMATCH[1]}" + # Skip test environments + if [[ ! "$env_name" =~ -test$ && ! "$env_name" =~ -all- ]]; then + envs+=("$env_name") + fi + fi + done < "$file" + fi + done + + # Sort and display unique environments + if [[ ${#envs[@]} -gt 0 ]]; then + printf '%s\n' "${envs[@]}" | sort -u | column -c 80 + else + log_warn "No environments found in configuration files" + fi + + echo "" + log_info "Total: $(printf '%s\n' "${envs[@]}" | sort -u | wc -l) environments" +} + +# Show usage +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Complete CI/CD build pipeline wrapper. + +Arguments: + environment PlatformIO environment name + +Options: + --version [TAG] Set version tag (if TAG omitted, auto-generated) + --mode MODE Build mode: 'prod' or 'dev' [default: prod] + 'dev' enables OTA and development version + --deploy-ready Prepare for deployment (renamed artifacts) + --output DIR Output directory for artifacts [default: generated/artifacts/] + --skip-verification Skip build tools verification + --clean Clean build before starting + --verbose Enable verbose output + --list-envs List all available PlatformIO environments + --help Show this help message + +Environment Variables: + CI Set to 'true' in CI/CD environments + BUILD_NUMBER Build number from CI/CD + GIT_COMMIT Git commit hash for versioning + +Examples: + # List available environments + $0 --list-envs + + # Local development build + $0 esp32dev-all-test --mode dev + + # Production release build + $0 esp32dev-bt --version v1.7.0 --mode prod --deploy-ready + + # CI/CD build (auto-detects version) + $0 theengs-bridge --version --mode dev + +EOF +} + +# Main pipeline +main() { + local environment="" + local version="" + local set_version=false + local mode="" + local deploy=false + local output_dir="" + local skip_verification=false + local clean=false + local verbose=false + + # Parse arguments + while [[ $# -gt 0 ]]; do + case "$1" in + --version) + set_version=true + # Check if next argument is a version tag or another option + if [[ $# -gt 1 && ! "$2" =~ ^-- ]]; then + version="$2" + shift 2 + else + shift + fi + ;; + --mode) + if [[ $# -lt 2 ]]; then + log_error "--mode requires an argument: 'prod' or 'dev'" + usage + exit 1 + fi + if [[ "$2" != "prod" && "$2" != "dev" ]]; then + log_error "Invalid mode: $2. Must be 'prod' or 'dev'" + usage + exit 1 + fi + mode="$2" + shift 2 + ;; + --deploy-ready) + deploy=true + shift + ;; + --output) + if [[ $# -lt 2 ]]; then + log_error "--output requires a directory argument" + usage + exit 1 + fi + output_dir="$2" + shift 2 + ;; + --skip-verification) + skip_verification=true + shift + ;; + --clean) + clean=true + shift + ;; + --verbose) + verbose=true + shift + ;; + --list-envs) + list_environments + exit 0 + ;; + --help|-h) + usage + exit 0 + ;; + -*) + log_error "Unknown option: $1" + usage + exit 1 + ;; + *) + environment="$1" + shift + ;; + esac + done + + # Set default mode if not specified + if [[ -z "$mode" ]]; then + mode="prod" + log_info "Mode not specified, defaulting to production" + fi + + # Validate environment + if [[ -z "$environment" ]]; then + log_error "Environment name is required" + usage + exit 1 + fi + + # Auto-generate version if --version flag is set but no tag provided + if [[ "$set_version" == "true" && -z "$version" ]]; then + if [[ "${CI:-false}" == "true" ]]; then + # CI/CD environment + version="${BUILD_NUMBER:-${GIT_COMMIT:-unknown}}" + else + # Local development + version="local-$(date +%Y%m%d-%H%M%S)" + fi + log_info "Auto-generated version: $version" + fi + + # Setup error handling + trap cleanup_on_error ERR + + # Change to project root + cd "$PROJECT_ROOT" + + # Start timer + local start_time + start_time=$(date +%s) + + # Print banner + print_banner + + # Step 1: Verify build tools + if [[ "$skip_verification" == "false" ]]; then + log_step "1/4 Verifying build tools..." + verify_build_tools || exit 1 + echo "" + else + log_warn "Skipping build tools verification (--skip-verification)" + echo "" + fi + + # Step 2: Set version (only if --version flag was provided) + if [[ "$set_version" == "true" ]]; then + log_step "2/4 Setting version: $version" + if [[ "$mode" == "dev" ]]; then + "${SCRIPT_DIR}/ci_set_version.sh" "$version" --dev || exit 1 + else + "${SCRIPT_DIR}/ci_set_version.sh" "$version" --prod || exit 1 + fi + echo "" + else + log_info "Skipping version setting (use --version to set version)" + echo "" + fi + + # Step 3: Build firmware + log_step "3/4 Building firmware for: $environment" + local build_opts=() + [[ "$mode" == "dev" ]] && build_opts+=(--dev-ota) + [[ "$clean" == "true" ]] && build_opts+=(--clean) + [[ "$verbose" == "true" ]] && build_opts+=(--verbose) + + "${SCRIPT_DIR}/ci_build_firmware.sh" "$environment" "${build_opts[@]}" || exit 1 + echo "" + + # Step 4: Prepare artifacts + log_step "4/4 Preparing artifacts..." + local artifact_opts=() + [[ "$deploy" == "true" ]] && artifact_opts+=(--deploy) || artifact_opts+=(--standard) + artifact_opts+=(--manifest) + [[ -n "$output_dir" ]] && artifact_opts+=(--output "$output_dir") + + "${SCRIPT_DIR}/ci_prepare_artifacts.sh" "$environment" "${artifact_opts[@]}" || exit 1 + echo "" + + # Print summary + print_summary "$environment" "$version" "$start_time" + + log_info "✓ Complete build pipeline finished successfully" +} + +# Run main if executed directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/scripts/ci_build_firmware.sh b/scripts/ci_build_firmware.sh new file mode 100755 index 00000000..5731c251 --- /dev/null +++ b/scripts/ci_build_firmware.sh @@ -0,0 +1,281 @@ +#!/bin/bash +# Builds firmware for specified PlatformIO environment +# Used by: CI/CD pipelines and local development +# Usage: ./build_firmware.sh [OPTIONS] + +set -euo pipefail + +# Constants +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +# Load shared configuration (colors, logging functions, paths) +if [[ -f "${SCRIPT_DIR}/ci_00_config.sh" ]]; then + source "${SCRIPT_DIR}/ci_00_config.sh" +else + echo "ERROR: ci_00_config.sh not found" >&2 + exit 1 +fi + +# Set absolute path for BUILD_DIR +BUILD_DIR="${PROJECT_ROOT}/${BUILD_DIR}" + +# Script-specific logging function +log_build() { echo -e "${BLUE}[BUILD]${NC} $*"; } + +# Function to validate environment name +validate_environment() { + local env="$1" + + if [[ -z "$env" ]]; then + log_error "Environment name is required" + return 1 + fi + + # Check if environment exists in platformio.ini or environments.ini + if ! grep -q "^\[env:${env}\]" "${PROJECT_ROOT}/platformio.ini" "${PROJECT_ROOT}/environments.ini" 2>/dev/null; then + log_warn "Environment '${env}' not found in configuration files" + log_warn "Proceeding anyway (PlatformIO will validate)" + fi + + log_info "Building environment: $env" +} + +# Function to setup build environment variables +setup_build_env() { + local enable_dev_ota="${1:-false}" + + export PYTHONIOENCODING=utf-8 + export PYTHONUTF8=1 + + if [[ "$enable_dev_ota" == "true" ]]; then + export PLATFORMIO_BUILD_FLAGS='"-DDEVELOPMENTOTA=true"' + log_info "Development OTA enabled" + fi +} + +# Function to check PlatformIO availability +check_platformio() { + if ! command -v platformio >/dev/null 2>&1; then + log_error "PlatformIO not found. Run setup_build_env.sh first" + return 1 + fi +} + +# Function to clean build artifacts +clean_build() { + local env="$1" + local env_dir="${BUILD_DIR}/${env}" + + if [[ -d "$env_dir" ]]; then + log_info "Cleaning previous build artifacts for: $env" + rm -rf "$env_dir" + fi +} + +# Function to run PlatformIO build +run_build() { + local env="$1" + local clean="${2:-false}" + local verbose="${3:-false}" + + log_build "Starting build for environment: $env" + + local build_cmd="platformio run -e $env" + + if [[ "$clean" == "true" ]]; then + build_cmd="platformio run -e $env --target clean && $build_cmd" + fi + + if [[ "$verbose" == "true" ]]; then + build_cmd="$build_cmd --verbose" + fi + + # Execute build with timing + local start_time + start_time=$(date +%s) + + if eval "$build_cmd"; then + local end_time + end_time=$(date +%s) + local duration=$((end_time - start_time)) + + log_build "Build completed successfully in ${duration}s" + return 0 + else + log_error "Build failed for environment: $env" + return 1 + fi +} + +# Function to verify build artifacts +verify_artifacts() { + local env="$1" + local env_dir="${BUILD_DIR}/${env}" + + log_info "Verifying build artifacts..." + + local artifacts_found=0 + local firmware="${env_dir}/firmware.bin" + local partitions="${env_dir}/partitions.bin" + local bootloader="${env_dir}/bootloader.bin" + + if [[ -f "$firmware" ]]; then + local size + size=$(stat -f%z "$firmware" 2>/dev/null || stat -c%s "$firmware" 2>/dev/null) + log_info "✓ firmware.bin (${size} bytes)" + ((artifacts_found++)) + else + log_warn "✗ firmware.bin not found" + fi + + if [[ -f "$partitions" ]]; then + log_info "✓ partitions.bin" + ((artifacts_found++)) + fi + + if [[ -f "$bootloader" ]]; then + log_info "✓ bootloader.bin" + ((artifacts_found++)) + fi + + if [[ $artifacts_found -eq 0 ]]; then + log_error "No build artifacts found" + return 1 + fi + + log_info "Found ${artifacts_found} artifact(s)" +} + +# Function to show build summary +show_build_summary() { + local env="$1" + local env_dir="${BUILD_DIR}/${env}" + + echo "" + echo "═══════════════════════════════════════" + echo " Build Summary: $env" + echo "═══════════════════════════════════════" + + if [[ -d "$env_dir" ]]; then + find "$env_dir" -name "*.bin" -o -name "*.elf" | while read -r file; do + local size + size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file" 2>/dev/null) + local size_kb=$((size / 1024)) + echo " $(basename "$file"): ${size_kb} KB" + done + fi + + echo "═══════════════════════════════════════" +} + +# Show usage +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Build firmware for a specific PlatformIO environment. + +Arguments: + environment PlatformIO environment name (e.g., esp32dev-all-test) + +Options: + --dev-ota Enable development OTA build flags + --clean Clean build artifacts before building + --verbose Enable verbose build output + --no-verify Skip artifact verification + --help Show this help message + +Examples: + $0 esp32dev-all-test + $0 esp32dev-bt --dev-ota + $0 theengs-bridge --clean --verbose + +EOF +} + +# Main execution +main() { + local environment="" + local enable_dev_ota=false + local clean_build_flag=false + local verbose=false + local verify=true + + # Parse arguments + while [[ $# -gt 0 ]]; do + case "$1" in + --dev-ota) + enable_dev_ota=true + shift + ;; + --clean) + clean_build_flag=true + shift + ;; + --verbose) + verbose=true + shift + ;; + --no-verify) + verify=false + shift + ;; + --help|-h) + usage + exit 0 + ;; + -*) + log_error "Unknown option: $1" + usage + exit 1 + ;; + *) + environment="$1" + shift + ;; + esac + done + + # Validate inputs + if [[ -z "$environment" ]]; then + log_error "Environment name is required" + usage + exit 1 + fi + + # Change to project root + cd "$PROJECT_ROOT" + + # Check prerequisites + check_platformio || exit 1 + + # Validate environment + validate_environment "$environment" || exit 1 + + # Setup build environment + setup_build_env "$enable_dev_ota" + + # Clean if requested + if [[ "$clean_build_flag" == "true" ]]; then + clean_build "$environment" + fi + + # Run build + run_build "$environment" "$clean_build_flag" "$verbose" || exit 1 + + # Verify artifacts + if [[ "$verify" == "true" ]]; then + verify_artifacts "$environment" || exit 1 + fi + + # Show summary + show_build_summary "$environment" + + log_info "Build process completed successfully" +} + +# Run main if executed directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/scripts/ci_prepare_artifacts.sh b/scripts/ci_prepare_artifacts.sh new file mode 100755 index 00000000..e78f7849 --- /dev/null +++ b/scripts/ci_prepare_artifacts.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# Prepares firmware artifacts for upload or deployment +# Used by: CI/CD pipelines for artifact packaging +# Usage: ./prepare_artifacts.sh [OPTIONS] + +set -euo pipefail + +# Constants +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +# Load shared configuration (colors, logging functions, paths) +if [[ -f "${SCRIPT_DIR}/ci_00_config.sh" ]]; then + source "${SCRIPT_DIR}/ci_00_config.sh" +else + echo "ERROR: ci_00_config.sh not found" >&2 + exit 1 +fi + +# Set absolute paths +BUILD_DIR="${PROJECT_ROOT}/${BUILD_DIR}" +DEFAULT_OUTPUT_DIR="${PROJECT_ROOT}/${ARTIFACTS_DIR}" + +# Function to create output directory +create_output_dir() { + local output_dir="$1" + + if [[ -d "$output_dir" ]]; then + log_warn "Output directory already exists: $output_dir" + log_info "Cleaning existing artifacts..." + rm -rf "$output_dir" + fi + + mkdir -p "$output_dir" + log_info "Created output directory: $output_dir" +} + +# Function to copy artifact with optional renaming +copy_artifact() { + local source="$1" + local dest="$2" + local artifact_type="$3" + + if [[ ! -f "$source" ]]; then + log_warn "${artifact_type} not found: $source" + return 1 + fi + + if cp "$source" "$dest"; then + local size + size=$(stat -f%z "$dest" 2>/dev/null || stat -c%s "$dest" 2>/dev/null) + local size_kb=$((size / 1024)) + log_info "✓ Copied ${artifact_type}: $(basename "$dest") (${size_kb} KB)" + return 0 + else + log_error "Failed to copy ${artifact_type}: $source" + return 1 + fi +} + +# Function to prepare standard artifacts (no renaming) +prepare_standard_artifacts() { + local env="$1" + local output_dir="$2" + local env_dir="${BUILD_DIR}/${env}" + + log_info "Preparing STANDARD artifacts for: $env" + + local copied=0 + + # Copy firmware.bin (required) + if copy_artifact "${env_dir}/firmware.bin" "${output_dir}/firmware.bin" "firmware"; then + ((copied++)) + fi + + # Copy partitions.bin (optional) + copy_artifact "${env_dir}/partitions.bin" "${output_dir}/partitions.bin" "partitions" && ((copied++)) || true + + # Note: bootloader.bin is NOT copied in standard mode (only needed for deployment) + + if [[ $copied -eq 0 ]]; then + log_error "No artifacts were copied" + return 1 + fi + + log_info "Copied ${copied} artifact(s) in standard mode" +} + +# Function to prepare deployment artifacts (with renaming) +prepare_deployment_artifacts() { + local env="$1" + local output_dir="$2" + local env_dir="${BUILD_DIR}/${env}" + + log_info "Preparing DEPLOYMENT artifacts for: $env" + + local copied=0 + + # Copy and rename firmware.bin + if copy_artifact "${env_dir}/firmware.bin" "${output_dir}/${env}-firmware.bin" "firmware"; then + ((copied++)) + fi + + # Copy and rename partitions.bin (optional) + copy_artifact "${env_dir}/partitions.bin" "${output_dir}/${env}-partitions.bin" "partitions" && ((copied++)) || true + + # Copy and rename bootloader.bin (optional) + copy_artifact "${env_dir}/bootloader.bin" "${output_dir}/${env}-bootloader.bin" "bootloader" && ((copied++)) || true + + # Copy boot_app0.bin if exists (ESP32 specific) + copy_artifact "${env_dir}/boot_app0.bin" "${output_dir}/${env}-boot_app0.bin" "boot_app0" && ((copied++)) || true + + if [[ $copied -eq 0 ]]; then + log_error "No artifacts were copied" + return 1 + fi + + log_info "Copied ${copied} artifact(s) in deployment mode" +} + +# Function to create manifest file +create_manifest() { + local env="$1" + local output_dir="$2" + local manifest="${output_dir}/manifest.txt" + + log_info "Creating artifact manifest..." + + { + echo "Environment: $env" + echo "Build Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" + echo "Build Host: $(hostname)" + echo "" + echo "Artifacts:" + + find "$output_dir" -type f -name "*.bin" | sort | while read -r file; do + local size + size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file" 2>/dev/null) + local size_kb=$((size / 1024)) + local md5sum_val + md5sum_val=$(md5sum "$file" 2>/dev/null | cut -d' ' -f1 || md5 -q "$file" 2>/dev/null) + echo " - $(basename "$file"): ${size_kb} KB (MD5: ${md5sum_val})" + done + } > "$manifest" + + log_info "Manifest created: $manifest" +} + +# Function to compress artifacts (optional) +compress_artifacts() { + local output_dir="$1" + local archive_name="$2" + + log_info "Compressing artifacts..." + + local archive="${output_dir}/${archive_name}.tar.gz" + + if tar -czf "$archive" -C "$output_dir" .; then + local size + size=$(stat -f%z "$archive" 2>/dev/null || stat -c%s "$archive" 2>/dev/null) + local size_kb=$((size / 1024)) + log_info "Archive created: ${archive_name}.tar.gz (${size_kb} KB)" + else + log_error "Failed to create archive" + return 1 + fi +} + +# Function to list artifacts +list_artifacts() { + local output_dir="$1" + + echo "" + echo "═══════════════════════════════════════" + echo " Prepared Artifacts" + echo "═══════════════════════════════════════" + + if [[ -d "$output_dir" ]]; then + find "$output_dir" -type f | sort | while read -r file; do + local size + size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file" 2>/dev/null) + local size_kb=$((size / 1024)) + echo " $(basename "$file"): ${size_kb} KB" + done + else + echo " No artifacts found" + fi + + echo "═══════════════════════════════════════" +} + +# Show usage +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Prepare firmware artifacts for upload or deployment. + +Arguments: + environment PlatformIO environment name + +Options: + --deploy Prepare for deployment (rename with environment prefix) + --standard Prepare standard artifacts (no renaming) [default] + --output DIR Output directory [default: generated/artifacts/] + --manifest Create manifest file with artifact metadata + --compress Compress artifacts into tar.gz archive + --help Show this help message + +Examples: + $0 esp32dev-all-test + $0 esp32dev-bt --deploy --manifest + $0 theengs-bridge --output build/artifacts --compress + +EOF +} + +# Main execution +main() { + local environment="" + local mode="standard" + local output_dir="$DEFAULT_OUTPUT_DIR" + local create_manifest_flag=false + local compress_flag=false + + # Parse arguments + while [[ $# -gt 0 ]]; do + case "$1" in + --deploy) + mode="deploy" + shift + ;; + --standard) + mode="standard" + shift + ;; + --output) + output_dir="$2" + shift 2 + ;; + --manifest) + create_manifest_flag=true + shift + ;; + --compress) + compress_flag=true + shift + ;; + --help|-h) + usage + exit 0 + ;; + -*) + log_error "Unknown option: $1" + usage + exit 1 + ;; + *) + environment="$1" + shift + ;; + esac + done + + # Validate inputs + if [[ -z "$environment" ]]; then + log_error "Environment name is required" + usage + exit 1 + fi + + # Change to project root + cd "$PROJECT_ROOT" + + # Check if build directory exists + if [[ ! -d "${BUILD_DIR}/${environment}" ]]; then + log_error "Build directory not found for environment: $environment" + log_error "Run build_firmware.sh first" + exit 1 + fi + + # Create output directory + create_output_dir "$output_dir" + + # Prepare artifacts based on mode + case "$mode" in + standard) + prepare_standard_artifacts "$environment" "$output_dir" || exit 1 + ;; + deploy) + prepare_deployment_artifacts "$environment" "$output_dir" || exit 1 + ;; + *) + log_error "Unknown mode: $mode" + exit 1 + ;; + esac + + # Create manifest if requested + if [[ "$create_manifest_flag" == "true" ]]; then + create_manifest "$environment" "$output_dir" + fi + + # Compress if requested + if [[ "$compress_flag" == "true" ]]; then + compress_artifacts "$output_dir" "$environment" + fi + + # Show summary + list_artifacts "$output_dir" + + log_info "Artifact preparation completed successfully" +} + +# Run main if executed directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/scripts/ci_qa.sh b/scripts/ci_qa.sh new file mode 100755 index 00000000..ba4122c3 --- /dev/null +++ b/scripts/ci_qa.sh @@ -0,0 +1,409 @@ +#!/bin/bash +# CI/CD Quality Assurance (QA) - Code Linting and Formatting +# Checks and fixes code formatting using clang-format +# Usage: ./scripts/ci_qa.sh [OPTIONS] + +set -euo pipefail + +# Constants +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" + +# Load shared configuration +if [[ -f "${SCRIPT_DIR}/ci_00_config.sh" ]]; then + source "${SCRIPT_DIR}/ci_00_config.sh" +else + echo "ERROR: ci_00_config.sh not found" >&2 + exit 1 +fi + +# Default values +CHECK_MODE=true +FIX_MODE=false +FORMAT_ONLY=false +SOURCE_DIR="main" +EXTENSIONS="h,ino,cpp" +CLANG_FORMAT_VERSION="9" +VERBOSE=false + +# Function to check if clang-format is available +check_clang_format() { + local version="$1" + local cmd="clang-format-${version}" + + if command -v "$cmd" >/dev/null 2>&1; then + echo "$cmd" + return 0 + fi + + # Try without version suffix + if command -v clang-format >/dev/null 2>&1; then + echo "clang-format" + return 0 + fi + + return 1 +} + +# Function to find files to check +find_files() { + local source="$1" + local extensions="$2" + + if [[ ! -d "${PROJECT_ROOT}/${source}" ]]; then + return 1 + fi + + local find_patterns=() + IFS=',' read -ra exts <<< "$extensions" + for ext in "${exts[@]}"; do + find_patterns+=(-name "*.${ext}" -o) + done + # Remove last -o + unset 'find_patterns[-1]' + + local files + files=$(find "${PROJECT_ROOT}/${source}" -type f \( "${find_patterns[@]}" \) 2>/dev/null || true) + + if [[ -z "$files" ]]; then + return 1 + fi + + echo "$files" +} + +# Function to check formatting +check_formatting() { + local clang_format_cmd="$1" + local files="$2" + + log_info "Checking code formatting..." + + local failed_files=() + local checked_count=0 + local has_issues=false + + while IFS= read -r file; do + if [[ -z "$file" ]]; then + continue + fi + + checked_count=$((checked_count + 1)) + + if [[ "$VERBOSE" == true ]]; then + log_info "Checking: $file" + fi + + # Check if file needs formatting and capture diff + local diff_output + diff_output=$("$clang_format_cmd" --dry-run --Werror "$file" 2>&1) + local format_result=$? + + if [[ $format_result -ne 0 ]]; then + failed_files+=("$file") + has_issues=true + + # Show the actual formatting differences + echo "" + log_warn "⚠ Formatting issues in: $file" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + + # Generate and show diff with colors + local actual_diff + actual_diff=$(diff -u "$file" <("$clang_format_cmd" "$file") 2>/dev/null || true) + + if [[ -n "$actual_diff" ]]; then + echo "$actual_diff" | head -50 | while IFS= read -r line; do + if [[ "$line" =~ ^-[^-] ]]; then + echo -e "\033[31m$line\033[0m" # Red for removed lines + elif [[ "$line" =~ ^+[^+] ]]; then + echo -e "\033[32m$line\033[0m" # Green for added lines + elif [[ "$line" =~ ^@@ ]]; then + echo -e "\033[36m$line\033[0m" # Cyan for line numbers + else + echo "$line" + fi + done + else + echo "$diff_output" + fi + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + else + if [[ "$VERBOSE" == true ]]; then + log_info " ✓ OK" + fi + fi + done <<< "$files" + + echo "" + log_info "Checked ${checked_count} files" + + if [[ $has_issues == true ]]; then + echo "" + log_error "Found ${#failed_files[@]} files with formatting issues:" + for file in "${failed_files[@]}"; do + log_error " - $file" + done + echo "" + log_error "To fix these issues automatically, run:" + log_error " $0 --fix" + echo "" + return 1 + fi + + log_info "✓ All files are properly formatted" + return 0 +} + +# Function to fix formatting +fix_formatting() { + local clang_format_cmd="$1" + local files="$2" + + log_info "Fixing code formatting..." + + local fixed_count=0 + local total_count=0 + + while IFS= read -r file; do + if [[ -z "$file" ]]; then + continue + fi + + total_count=$((total_count + 1)) + + if [[ "$VERBOSE" == true ]]; then + log_info "Processing: $file" + fi + + # Apply formatting in-place + if "$clang_format_cmd" -i "$file" 2>/dev/null; then + fixed_count=$((fixed_count + 1)) + if [[ "$VERBOSE" == true ]]; then + log_info " ✓ Formatted" + fi + else + if [[ "$VERBOSE" == true ]]; then + log_warn " ✗ Failed to format" + fi + fi + done <<< "$files" + + echo "" + log_info "Processed ${total_count} files" + log_info "✓ Formatting applied to ${fixed_count} files" + + if [[ $fixed_count -gt 0 ]]; then + log_warn "" + log_warn "Files have been modified. Please review and commit the changes:" + log_warn " git diff" + log_warn " git add -u" + log_warn " git commit -m 'style: apply clang-format'" + fi +} + +# Function to run all QA checks +run_all_checks() { + log_info "Running all quality assurance checks..." + + local all_passed=true + + # Format check + log_info "═══ Code Formatting ═══" + if ! run_format_check; then + all_passed=false + fi + echo "" + + # Future: Add more checks here + # - cppcheck static analysis + # - code complexity metrics + # - TODO/FIXME detection + # - license header validation + + if [[ "$all_passed" == false ]]; then + log_error "Some QA checks failed" + return 1 + fi + + log_info "✓ All QA checks passed" + return 0 +} + +# Function to run format check +run_format_check() { + log_info "Checking for clang-format version ${CLANG_FORMAT_VERSION}..." + + local clang_format_cmd + clang_format_cmd=$(check_clang_format "$CLANG_FORMAT_VERSION") + + if [[ $? -ne 0 ]] || [[ -z "$clang_format_cmd" ]]; then + log_error "clang-format not found" + log_error "Please install clang-format:" + log_error " Ubuntu/Debian: sudo apt-get install clang-format-${CLANG_FORMAT_VERSION}" + log_error " macOS: brew install clang-format" + return 1 + fi + + if [[ "$clang_format_cmd" == "clang-format-${CLANG_FORMAT_VERSION}" ]]; then + log_info "✓ clang-format-${CLANG_FORMAT_VERSION} found" + else + local installed_version + installed_version=$(clang-format --version | grep -oP '\d+\.\d+' | head -1 || echo "unknown") + log_warn "clang-format-${CLANG_FORMAT_VERSION} not found, using clang-format (version ${installed_version})" + fi + + log_info "Finding files in '${SOURCE_DIR}' with extensions: ${EXTENSIONS}" + + local files + files=$(find_files "$SOURCE_DIR" "$EXTENSIONS") + + if [[ $? -ne 0 ]] || [[ -z "$files" ]]; then + log_error "Source directory not found: ${PROJECT_ROOT}/${SOURCE_DIR}" + return 1 + fi + + local file_count + file_count=$(echo "$files" | wc -l) + log_info "Found ${file_count} files to check" + + if [[ "$FIX_MODE" == true ]]; then + fix_formatting "$clang_format_cmd" "$files" + else + check_formatting "$clang_format_cmd" "$files" + fi +} + +# Function to show usage +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Run quality assurance checks on OpenMQTTGateway code. + +Options: + --check Check formatting only (CI mode) [default] + --fix Fix formatting issues automatically + --format Run only format checks + --all Run all QA checks [default] + --source DIR Source directory to check [default: main] + --extensions EXTS File extensions (comma-separated) [default: h,ino,cpp] + --clang-format-version V clang-format version [default: 9] + --verbose Enable verbose output + --help Show this help message + +Examples: + # Check formatting (CI mode) + $0 --check + + # Fix formatting issues + $0 --fix + + # Check specific directory + $0 --check --source lib/LEDManager + + # Check with custom extensions + $0 --check --extensions h,cpp + + # Verbose output + $0 --check --verbose + +EOF + exit 0 +} + +# Parse command line arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --check) + CHECK_MODE=true + FIX_MODE=false + shift + ;; + --fix) + FIX_MODE=true + CHECK_MODE=false + shift + ;; + --format) + FORMAT_ONLY=true + shift + ;; + --all) + FORMAT_ONLY=false + shift + ;; + --source) + SOURCE_DIR="$2" + shift 2 + ;; + --extensions) + EXTENSIONS="$2" + shift 2 + ;; + --clang-format-version) + CLANG_FORMAT_VERSION="$2" + shift 2 + ;; + --verbose) + VERBOSE=true + shift + ;; + --help) + usage + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac + done +} + +# Main execution +main() { + local start_time + start_time=$(date +%s) + + parse_args "$@" + + log_info "Starting QA pipeline..." + + if [[ "$FIX_MODE" == true ]]; then + log_info "Mode: FIX (will modify files)" + else + log_info "Mode: CHECK (read-only)" + fi + + # Run checks + local result=0 + if [[ "$FORMAT_ONLY" == true ]]; then + run_format_check || result=$? + else + run_all_checks || result=$? + fi + + local end_time + end_time=$(date +%s) + local duration=$((end_time - start_time)) + + echo "" + echo "╔════════════════════════════════════════╗" + echo "║ QA Pipeline Summary ║" + echo "╚════════════════════════════════════════╝" + echo " Duration: ${duration}s" + + if [[ $result -eq 0 ]]; then + echo " Status: SUCCESS ✓" + echo "╚════════════════════════════════════════╝" + return 0 + else + echo " Status: FAILED ✗" + echo "╚════════════════════════════════════════╝" + return 1 + fi +} + +# Execute main function +main "$@" diff --git a/scripts/ci_set_version.sh b/scripts/ci_set_version.sh new file mode 100755 index 00000000..572bb8dc --- /dev/null +++ b/scripts/ci_set_version.sh @@ -0,0 +1,197 @@ +#!/bin/bash +# Updates version tags in firmware configuration and JSON files +# Used by: CI/CD pipelines for versioning builds +# Usage: ./set_version.sh [--dev] + +set -euo pipefail + +# Constants +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +readonly USER_CONFIG="${PROJECT_ROOT}/main/User_config.h" +readonly LATEST_VERSION_PROD="${SCRIPT_DIR}/latest_version.json" +readonly LATEST_VERSION_DEV="${SCRIPT_DIR}/latest_version_dev.json" + +# Load shared configuration (colors, logging functions, paths) +if [[ -f "${SCRIPT_DIR}/ci_00_config.sh" ]]; then + source "${SCRIPT_DIR}/ci_00_config.sh" +else + echo "ERROR: ci_00_config.sh not found" >&2 + exit 1 +fi + +# Function to validate version tag +validate_version() { + local version="$1" + + if [[ -z "$version" ]] || [[ "$version" == "version_tag" ]]; then + log_error "Invalid version tag: '$version'" + return 1 + fi + + log_info "Version tag validated: $version" +} + +# Function to backup files +backup_file() { + local file="$1" + + if [[ -f "$file" ]]; then + cp "$file" "${file}.bak" + log_info "Backed up: $file" + fi +} + +# Function to replace version in file +replace_version() { + local file="$1" + local version="$2" + + if [[ ! -f "$file" ]]; then + log_warn "File not found: $file (skipping)" + return 0 + fi + + # Backup before modification + backup_file "$file" + + # Replace version_tag placeholder + if sed -i "s/version_tag/${version}/g" "$file"; then + log_info "Updated version in: $file" + else + log_error "Failed to update version in: $file" + return 1 + fi +} + +# Function to set production version +set_production_version() { + local version="$1" + + log_info "Setting PRODUCTION version: $version" + + replace_version "$USER_CONFIG" "$version" || return 1 + replace_version "$VERSION_JSON" "$version" || return 1 +} + +# Function to set development version +set_development_version() { + local version="$1" + + log_info "Setting DEVELOPMENT version: $version" + + replace_version "$USER_CONFIG" "$version" || return 1 + replace_version "$VERSION_DEV_JSON" "$version" || return 1 +} + +# Function to restore backups +restore_backups() { + log_warn "Restoring backups..." + + for file in "$USER_CONFIG" "$VERSION_JSON" "$VERSION_DEV_JSON"; do + if [[ -f "${file}.bak" ]]; then + mv "${file}.bak" "$file" + log_info "Restored: $file" + fi + done +} + +# Function to clean backups +clean_backups() { + for file in "$USER_CONFIG" "$VERSION_JSON" "$VERSION_DEV_JSON"; do + if [[ -f "${file}.bak" ]]; then + rm "${file}.bak" + fi + done +} + +# Show usage +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Update version tags in firmware configuration files. + +Arguments: + version_tag Version string to inject (e.g., v1.2.3, abc123, dev-20230101) + +Options: + --dev Use development version files (latest_version_dev.json) + --prod Use production version files (latest_version.json) [default] + --help Show this help message + +Examples: + $0 v1.2.3 # Production release + $0 abc123 --dev # Development build + $0 \${{ github.sha }} --dev # CI/CD with commit SHA + +EOF +} + +# Main execution +main() { + local version="" + local is_dev=false + + # Parse arguments + while [[ $# -gt 0 ]]; do + case "$1" in + --dev) + is_dev=true + shift + ;; + --prod) + is_dev=false + shift + ;; + --help|-h) + usage + exit 0 + ;; + -*) + log_error "Unknown option: $1" + usage + exit 1 + ;; + *) + version="$1" + shift + ;; + esac + done + + # Validate version + if [[ -z "$version" ]]; then + log_error "Version tag is required" + usage + exit 1 + fi + + validate_version "$version" || exit 1 + + # Change to project root + cd "$PROJECT_ROOT" + + # Set version with error handling + if [[ "$is_dev" == true ]]; then + set_development_version "$version" || { + restore_backups + exit 1 + } + else + set_production_version "$version" || { + restore_backups + exit 1 + } + fi + + # Clean up backups on success + clean_backups + + log_info "Version update completed successfully" +} + +# Run main if executed directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/scripts/ci_site.sh b/scripts/ci_site.sh new file mode 100755 index 00000000..1b664e00 --- /dev/null +++ b/scripts/ci_site.sh @@ -0,0 +1,505 @@ +#!/bin/bash +# CI/CD Site/Documentation Builder +# Builds and deploys VuePress documentation with version management +# Usage: ./scripts/ci_site.sh [OPTIONS] + +set -euo pipefail + +# Constants +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +readonly DOCS_DIR="${PROJECT_ROOT}/docs" +readonly VUEPRESS_CONFIG="${DOCS_DIR}/.vuepress/config.js" +readonly VUEPRESS_BUILD_DIR="${DOCS_DIR}/.vuepress/dist" + +# Load shared configuration +if [[ -f "${SCRIPT_DIR}/ci_00_config.sh" ]]; then + source "${SCRIPT_DIR}/ci_00_config.sh" +else + echo "ERROR: ci_00_config.sh not found" >&2 + exit 1 +fi + +# Final output directory for site +readonly SITE_OUTPUT_DIR="${PROJECT_ROOT}/${SITE_DIR}" + +# Default values +MODE="prod" +URL_PREFIX="/" +CUSTOM_VERSION="" +VERSION_SOURCE="release" +PREVIEW=false +GENERATE_WEBUPLOADER=true +WEBUPLOADER_ARGS="" +DEPLOY_DIR="." +RUN_PAGESPEED=false +PAGESPEED_URL="https://docs.openmqttgateway.com/" + +# Function to check required tools +check_requirements() { + log_info "Checking required tools..." + + local missing_tools=() + + if ! command -v node >/dev/null 2>&1; then + missing_tools+=("node") + else + local node_version + node_version=$(node --version) + log_info "✓ Node.js ${node_version} found" + fi + + if ! command -v npm >/dev/null 2>&1; then + missing_tools+=("npm") + else + local npm_version + npm_version=$(npm --version) + log_info "✓ npm ${npm_version} found" + fi + + if ! command -v python3 >/dev/null 2>&1; then + missing_tools+=("python3") + else + local python_version + python_version=$(python3 --version | grep -oP '\d+\.\d+' || echo "unknown") + log_info "✓ Python ${python_version} found" + fi + + if ! command -v pip3 >/dev/null 2>&1; then + missing_tools+=("pip3") + else + local pip_version + pip_version=$(pip3 --version | grep -oP '\d+\.\d+' || echo "unknown") + log_info "✓ pip ${pip_version} found" + fi + + if [[ ${#missing_tools[@]} -gt 0 ]]; then + log_error "Missing required tools: ${missing_tools[*]}" + return 1 + fi + + log_info "All required tools are available" + return 0 +} + +# Function to install dependencies +install_dependencies() { + log_info "Installing dependencies..." + + # Upgrade pip first (if pip module is available) + if python3 -m pip --version >/dev/null 2>&1; then + log_info "Upgrading pip..." + python3 -m pip install --upgrade pip --quiet || { + log_warn "Failed to upgrade pip, continuing with existing version..." + } + fi + + # Install Python dependencies (without --user if in virtualenv) + log_info "Installing Python dependencies..." + pip3 install requests pandas markdown pytablereader tabulate || { + log_error "Failed to install Python dependencies" + return 1 + } + + # Install Node dependencies + log_info "Installing Node.js dependencies..." + cd "${PROJECT_ROOT}" + npm install --quiet || { + log_error "Failed to install Node.js dependencies" + return 1 + } + + log_info "Dependencies installed successfully" +} + +# Function to download common config +download_common_config() { + log_info "Downloading common configuration..." + + local config_url="https://www.theengs.io/commonConfig.js" + local config_dest="${DOCS_DIR}/.vuepress/public/commonConfig.js" + + mkdir -p "$(dirname "$config_dest")" + + if curl -sSf -o "$config_dest" "$config_url"; then + log_info "✓ Common config downloaded" + else + log_warn "Failed to download common config, continuing anyway..." + fi +} + +# Function to get version +get_version() { + local version="" + + if [[ "$VERSION_SOURCE" == "custom" && -n "$CUSTOM_VERSION" ]]; then + version="$CUSTOM_VERSION" + log_info "Using custom version: $version" + elif [[ "$VERSION_SOURCE" == "release" ]]; then + # Try to get latest git tag (simulating GitHub release) + if command -v git >/dev/null 2>&1 && [[ -d "${PROJECT_ROOT}/.git" ]]; then + version=$(git describe --tags --abbrev=0 2>/dev/null || echo "development") + log_info "Using release version from git: $version" + else + version="development" + log_warn "Git not available, using version: $version" + fi + else + # Auto-detect from git + if command -v git >/dev/null 2>&1 && [[ -d "${PROJECT_ROOT}/.git" ]]; then + version=$(git describe --tags --abbrev=0 2>/dev/null || echo "development") + log_info "Using git version: $version" + else + version="development" + log_warn "Git not available, using version: $version" + fi + fi + + echo "$version" +} + +# Function to set version in config files +set_version() { + local version="$1" + + log_info "Setting version: $version" + + # Update VuePress config + if [[ -f "$VUEPRESS_CONFIG" ]]; then + sed -i "s|version_tag|${version}|g" "$VUEPRESS_CONFIG" + fi + + # Update version JSON file based on version source + if [[ "$VERSION_SOURCE" == "custom" ]]; then + # Custom version updates dev file + local version_file="${SCRIPT_DIR}/latest_version_dev.json" + if [[ -f "$version_file" ]]; then + sed -i "s|version_tag|${version}|g" "$version_file" + fi + else + # Release version updates production file + local version_file="${SCRIPT_DIR}/latest_version.json" + if [[ -f "$version_file" ]]; then + sed -i "s|version_tag|${version}|g" "$version_file" + fi + fi +} + +# Function to set URL prefix (base path) +set_url_prefix() { + local url_prefix="$1" + + if [[ "$url_prefix" != "/" ]]; then + log_info "Setting URL prefix: $url_prefix" + sed -i "s|base: '/'|base: '${url_prefix}'|g" "$VUEPRESS_CONFIG" + fi +} + +# Function to generate board documentation +generate_board_docs() { + log_info "Generating board documentation..." + + local generator="${SCRIPT_DIR}/generate_board_docs.py" + + if [[ -f "$generator" ]]; then + cd "${PROJECT_ROOT}" + python3 "$generator" || { + log_warn "Board documentation generation failed, continuing..." + } + else + log_warn "Board documentation generator not found, skipping..." + fi +} + +# Function to generate WebUploader manifest +generate_webuploader() { + if [[ "$GENERATE_WEBUPLOADER" != true ]]; then + log_info "Skipping WebUploader generation" + return 0 + fi + + log_info "Generating WebUploader manifest..." + + local generator="${SCRIPT_DIR}/gen_wu.py" + + if [[ -f "$generator" ]]; then + cd "${PROJECT_ROOT}" + python3 "$generator" $WEBUPLOADER_ARGS || { + log_warn "WebUploader generation failed, continuing..." + } + else + log_warn "WebUploader generator not found, skipping..." + fi +} + +# Function to build documentation +build_docs() { + log_info "Building documentation..." + + cd "${PROJECT_ROOT}" + + # Set Node options for compatibility with newer Node.js versions + export NODE_OPTIONS="--openssl-legacy-provider" + + npm run docs:build || { + log_error "Documentation build failed" + return 1 + } + + if [[ -d "$VUEPRESS_BUILD_DIR" ]]; then + log_info "✓ Documentation built successfully" + + # Copy to centralized output directory + log_info "Copying site to: $SITE_OUTPUT_DIR" + mkdir -p "$SITE_OUTPUT_DIR" + rm -rf "${SITE_OUTPUT_DIR}"/* + cp -r "${VUEPRESS_BUILD_DIR}"/* "${SITE_OUTPUT_DIR}/" + + log_info " VuePress output: $VUEPRESS_BUILD_DIR" + log_info " Final output: $SITE_OUTPUT_DIR" + else + log_error "Build output directory not found: $VUEPRESS_BUILD_DIR" + return 1 + fi +} + +# Function to preview documentation +preview_docs() { + log_info "Starting documentation preview..." + + if [[ ! -d "$SITE_OUTPUT_DIR" ]]; then + log_error "Build output not found. Run build first." + return 1 + fi + + log_info "Preview server starting at: http://localhost:8080" + log_info "Press Ctrl+C to stop" + + cd "$SITE_OUTPUT_DIR" + python3 -m http.server 8080 +} + +# Show usage +usage() { + cat << EOF +Usage: $0 [OPTIONS] + +Build and deploy OpenMQTTGateway documentation site using VuePress. + +This script handles the complete documentation build pipeline including: +- Installing Node.js and Python dependencies +- Downloading shared configuration from theengs.io +- Version management (from git tags or custom version) +- Generating board documentation from environments +- Creating WebUploader manifest for firmware updates +- Building VuePress static site +- Optional local preview or deployment to GitHub Pages + +OPTIONS: + --mode MODE + Build mode: 'prod' or 'dev' [default: prod] + - prod: Production documentation with release version + - dev: Development documentation with custom version tag + + --url-prefix PATH + URL prefix for documentation routing (VuePress base path) [default: /] + Controls how URLs are generated in the built site. + Examples: + --url-prefix / # Production site at root URL + --url-prefix /dev/ # Development site at /dev/ URL path + Note: Should match --deploy-dir for correct link generation + + --custom-version TAG + Override version tag displayed in documentation + Example: --custom-version "v1.8.0-beta" + Note: Automatically sets --version-source to 'custom' + + --version-source SOURCE + Source for version information: 'release' or 'custom' [default: release] + - release: Use latest git tag from repository + - custom: Use version from --custom-version parameter + + --preview + Start local HTTP server to preview built documentation + Server will run at http://localhost:8080 + Press Ctrl+C to stop the preview server + + --deploy-dir DIR + Deployment directory on GitHub Pages [default: .] + Controls where files are copied in the gh-pages branch. + Examples: + --deploy-dir . # Deploy to root of gh-pages branch + --deploy-dir dev # Deploy to dev/ folder in gh-pages + Note: Should match --url-prefix for correct site structure + + --no-webuploader + Skip WebUploader manifest generation + By default, generates manifest for web-based firmware updates + + --webuploader-args ARGS + Additional arguments passed to gen_wu.py script + Example: --webuploader-args "--dev" + + --run-pagespeed + Run Google PageSpeed Insights after deployment + Requires APIKEY to be configured in workflow + + --pagespeed-url URL + URL to test with PageSpeed Insights + Default: https://docs.openmqttgateway.com/ + + --help + Show this help message and exit + +EXAMPLES: + + # Build production documentation with latest release version + $0 --mode prod + + # Build and preview development documentation locally + $0 --mode dev --url-prefix /dev/ --preview + + # Build with custom version tag + $0 --custom-version "v1.8.0-beta" --version-source custom + + # Build without WebUploader manifest + $0 --mode prod --no-webuploader + + # Build for dev environment with custom WebUploader args + $0 --mode dev --url-prefix /dev/ --deploy-dir dev --webuploader-args "--dev" + +WORKFLOW: + 1. Check requirements (Node.js, npm, Python) + 2. Install dependencies (npm packages, Python libraries) + 3. Download common configuration from theengs.io + 4. Determine version (from git tags or custom) + 5. Generate board documentation from PlatformIO environments + 6. Generate WebUploader manifest (optional) + 7. Build VuePress static site + 8. Preview (optional) + +OUTPUT: + Built documentation will be in: generated/site/ + +EOF + exit 0 +} + +# Parse command line arguments +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --mode) + MODE="$2" + shift 2 + ;; + --url-prefix|--base-path) + URL_PREFIX="$2" + shift 2 + ;; + --deploy-dir|--destination-dir) + DEPLOY_DIR="$2" + shift 2 + ;; + --custom-version) + CUSTOM_VERSION="$2" + VERSION_SOURCE="custom" + shift 2 + ;; + --version-source) + VERSION_SOURCE="$2" + shift 2 + ;; + --preview) + PREVIEW=true + shift + ;; + + --no-webuploader) + GENERATE_WEBUPLOADER=false + shift + ;; + --webuploader-args) + WEBUPLOADER_ARGS="$2" + shift 2 + ;; + --run-pagespeed) + RUN_PAGESPEED=true + shift + ;; + --pagespeed-url) + PAGESPEED_URL="$2" + shift 2 + ;; + --help) + usage + ;; + *) + log_error "Unknown option: $1" + usage + ;; + esac + done +} + +# Main execution +main() { + local start_time + start_time=$(date +%s) + + parse_args "$@" + + log_info "Starting site build pipeline..." + log_info "Mode: $MODE" + log_info "URL prefix: $URL_PREFIX" + log_info "Deploy directory: $DEPLOY_DIR" + + # Check requirements + check_requirements || exit 1 + + # Install dependencies + install_dependencies || exit 1 + + # Download common config + download_common_config + + # Get and set version + local version + version=$(get_version) + set_version "$version" + + # Set URL prefix + set_url_prefix "$URL_PREFIX" + + # Generate board documentation + generate_board_docs + + # Generate WebUploader manifest + generate_webuploader + + # Build documentation + build_docs || exit 1 + + # Preview if requested + if [[ "$PREVIEW" == true ]]; then + preview_docs + fi + + local end_time + end_time=$(date +%s) + local duration=$((end_time - start_time)) + + echo "" + echo "╔════════════════════════════════════════╗" + echo "║ Site Build Summary ║" + echo "╚════════════════════════════════════════╝" + echo " Mode: $MODE" + echo " Version: $version" + echo " Duration: ${duration}s" + echo " Output: $SITE_OUTPUT_DIR" + echo " Status: SUCCESS ✓" + echo "╚════════════════════════════════════════╝" +} + +# Execute main function +main "$@" diff --git a/scripts/common_wu.py b/scripts/common_wu.py index a580627a..3021d226 100644 --- a/scripts/common_wu.py +++ b/scripts/common_wu.py @@ -1,3 +1,5 @@ +# Common templates and constants for web installer manifest generation +# Used by: scripts/gen_wu.py import string mf_temp32 = string.Template('''{ diff --git a/scripts/compressFirmware.py b/scripts/compressFirmware.py index cd57a6d5..0616337b 100644 --- a/scripts/compressFirmware.py +++ b/scripts/compressFirmware.py @@ -1,3 +1,5 @@ +# Compresses firmware binaries with gzip for OTA updates during build +# Used by: PlatformIO environments (optional, commented in environments.ini) import gzip import shutil import os diff --git a/scripts/gen_wu.py b/scripts/gen_wu.py index f33157b0..e5465310 100644 --- a/scripts/gen_wu.py +++ b/scripts/gen_wu.py @@ -1,3 +1,5 @@ +# Creates web installer manifests for ESP Web Tools firmware installation +# Used by: .github/workflows/task-docs.yml import os import requests import json diff --git a/scripts/generate_board_docs.py b/scripts/generate_board_docs.py index 19a5d2e4..c44f044f 100644 --- a/scripts/generate_board_docs.py +++ b/scripts/generate_board_docs.py @@ -1,3 +1,5 @@ +# Generates board documentation table from platformio.ini environments +# Used by: .github/workflows/task-docs.yml import pytablereader as ptr import pandas as pd import os diff --git a/scripts/prepare_deploy.sh b/scripts/prepare_deploy.sh index 2dda51b3..90b603cd 100755 --- a/scripts/prepare_deploy.sh +++ b/scripts/prepare_deploy.sh @@ -1,4 +1,6 @@ #!/bin/bash +# Prepares firmware binaries and libraries for GitHub release deployment +# Used by: .github/workflows/release.yml set -e echo "renaming bin files with the environment name" rename -v 's:/:-:g' .pio/build/*/*.bin diff --git a/scripts/replace_lib.py b/scripts/replace_lib.py index a792cc5c..9e69e2dd 100644 --- a/scripts/replace_lib.py +++ b/scripts/replace_lib.py @@ -1,4 +1,5 @@ - +# Replaces BLE library with custom version during PlatformIO build +# Used by: Currently unused (utility script for BLE library replacement) import shutil import os import hashlib