Test Action to build release

Author: MikeSoperRubicon

.github/workflows/push.yml

@@ -1,55 +1,27 @@
-name: ESP32 Arduino Libs CI
-
+name: IDF v5.3 with P4
 on:
-  push:
-    branches:
-      - master
-      - release/v5.3
-  pull_request:
-    paths:
-      - "**"
-      - "!**.md"
-      - "!.github/workflows/cron_build.yml"
-      - "!.github/workflows/cron.yml"
-      - "!.github/workflows/docker.yml"
-      - "!.github/workflows/repository_dispatch.yml"
-      - "!tools/config_editor/**"
-      - "!tools/docker/**"
-
-concurrency:
-  group: esp-idf-libs-${{github.event.pull_request.number || github.ref}}
-  cancel-in-progress: true
+  workflow_dispatch: # Manually start a workflow
 
 jobs:
   build-libs:
-    name: Build Libs for ${{ matrix.target }}
+    name: Build Arduino Libs
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        target: [esp32, esp32s3]
-      fail-fast: false
     steps:
       - uses: actions/checkout@v4
-
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
       - name: Install dependencies
         run: bash ./tools/prepare-ci.sh
-
-      - name: Build Libs for ${{ matrix.target }}
-        run: bash ./build.sh -e -t ${{ matrix.target }}
-
-      - name: Upload build
-        if: failure()
-        uses: actions/upload-artifact@v4
-        with:
-          name: build-${{ matrix.target }}
-          path: build
-
-      - name: Upload archive
-        uses: actions/upload-artifact@v4
-        with:
-          name: artifacts-${{ matrix.target }}
-          path: dist
-
+      - name: Get current branch
+        run: |
+          echo "GIT_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_ENV
+      - name: Build Arduino Libs
+        env:
+          GITHUB_TOKEN: ${{ secrets.PUSH_TOKEN }}
+          GIT_AUTHOR_EMAIL: ${{ secrets.PUSH_EMAIL }}
+        run: bash ./build.sh
       - name: Release
         uses: jason2866/action-gh-release@v1.3
         with:
@@ -59,34 +31,5 @@ jobs:
           files: |
             dist/framework*
             release-info.txt
-
-  combine-artifacts:
-    name: Combine artifacts
-    needs: build-libs
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: dist
-          pattern: artifacts-*
-          merge-multiple: true
-
-      - shell: bash
-        run: |
-          mkdir -p out
-          find dist -name 'arduino-esp32-libs-esp*.tar.gz' -exec tar zxvf {} -C out \;
-          cd out/tools/esp32-arduino-libs && tar zcf ../../../dist/esp32-arduino-libs.tar.gz * && cd ../../..
-          cp out/package_esp32_index.template.json dist/package_esp32_index.template.json
-
-      - name: Upload full esp32-arduino-libs archive
-        uses: actions/upload-artifact@v4
-        with:
-          name: esp32-arduino-libs
-          path: dist/esp32-arduino-libs.tar.gz
-
-      - name: Upload package_esp32_index.template.json
-        uses: actions/upload-artifact@v4
-        with:
-          name: package-esp32-index-json
-          path: dist/package_esp32_index.template.json
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

build.sh

@@ -20,51 +20,38 @@ export IDF_COMPONENT_OVERWRITE_MANAGED_COMPONENTS=1
 
 CCACHE_ENABLE=1
 
-TARGET="all"
+export TARGET="all"
 BUILD_TYPE="all"
-BUILD_DEBUG="default"
 SKIP_ENV=0
 COPY_OUT=0
-ARCHIVE_OUT=0
-if [ -z $DEPLOY_OUT ]; then
-    DEPLOY_OUT=0
-fi
+ARCHIVE_OUT=1
+DEPLOY_OUT=0
 
 function print_help() {
-    echo "Usage: build.sh [-s] [-n] [-A <arduino_branch>] [-I <idf_branch>] [-D <debug_level>] [-i <idf_commit>] [-c <path>] [-t <target>] [-b <build|menuconfig|reconfigure|idf-libs|copy-bootloader|mem-variant>] [config ...]"
+    echo "Usage: build.sh [-s] [-n] [-A <arduino_branch>] [-I <idf_branch>] [-i <idf_commit>] [-c <path>] [-t <target>] [-b <build|menuconfig|reconfigure|idf-libs|copy-bootloader|mem-variant>] [config ...]"
     echo "       -s     Skip installing/updating of ESP-IDF and all components"
     echo "       -n     Disable ccache"
     echo "       -A     Set which branch of arduino-esp32 to be used for compilation"
     echo "       -I     Set which branch of ESP-IDF to be used for compilation"
     echo "       -i     Set which commit of ESP-IDF to be used for compilation"
     echo "       -e     Archive the build to dist"
-    echo "       -d     Deploy the build to github arduino-esp32"
-    echo "       -D     Debug level to be set to ESP-IDF. One of default,none,error,warning,info,debug or verbose"
-    echo "       -c     Set the arduino-esp32 folder to copy the result to. ex. '$HOME/Arduino/hardware/espressif/esp32'"
     echo "       -t     Set the build target(chip) ex. 'esp32s3' or select multiple targets(chips) by separating them with comma ex. 'esp32,esp32s3,esp32c3'"
     echo "       -b     Set the build type. ex. 'build' to build the project and prepare for uploading to a board"
     echo "       ...    Specify additional configs to be applied. ex. 'qio 80m' to compile for QIO Flash@80MHz. Requires -b"
     exit 1
 }
 
-while getopts ":A:I:i:c:t:b:D:sde" opt; do
+while getopts ":A:I:i:c:t:b:sde" opt; do
     case ${opt} in
         s )
             SKIP_ENV=1
             ;;
         n )
             CCACHE_ENABLE=0
             ;;
-        d )
-            DEPLOY_OUT=1
-            ;;
         e )
             ARCHIVE_OUT=1
             ;;
-        c )
-            export ESP32_ARDUINO="$OPTARG"
-            COPY_OUT=1
-            ;;
         A )
             export AR_BRANCH="$OPTARG"
             ;;
@@ -74,19 +61,16 @@ while getopts ":A:I:i:c:t:b:D:sde" opt; do
         i )
             export IDF_COMMIT="$OPTARG"
             ;;
-        D )
-            BUILD_DEBUG="$OPTARG"
-            ;;
         t )
             IFS=',' read -ra TARGET <<< "$OPTARG"
             ;;
         b )
             b=$OPTARG
-            if [ "$b" != "build" ] && 
-               [ "$b" != "menuconfig" ] && 
-               [ "$b" != "reconfigure" ] && 
-               [ "$b" != "idf-libs" ] && 
-               [ "$b" != "copy-bootloader" ] && 
+            if [ "$b" != "build" ] &&
+               [ "$b" != "menuconfig" ] &&
+               [ "$b" != "reconfigure" ] &&
+               [ "$b" != "idf-libs" ] &&
+               [ "$b" != "copy-bootloader" ] &&
                [ "$b" != "mem-variant" ]; then
                 print_help
             fi
@@ -111,6 +95,7 @@ export IDF_CCACHE_ENABLE=$CCACHE_ENABLE
 echo "TARGET(s): ${TARGET[@]}"
 
 mkdir -p dist
+rm -rf dependencies.lock
 
 if [ $SKIP_ENV -eq 0 ]; then
     echo "* Installing/Updating ESP-IDF and all components..."
@@ -155,7 +140,7 @@ if [ "$BUILD_TYPE" != "all" ]; then
             continue
         fi
 
-        configs="configs/defconfig.common;configs/defconfig.$target;configs/defconfig.debug_$BUILD_DEBUG"
+        configs="configs/defconfig.common;configs/defconfig.$target"
         for defconf in `echo "$target_json" | jq -c '.features[]' | tr -d '"'`; do
             configs="$configs;configs/defconfig.$defconf"
         done
@@ -178,6 +163,15 @@ fi
 rm -rf build sdkconfig out
 mkdir -p "$AR_TOOLS/esp32-arduino-libs"
 
+# Add release-info
+rm -rf release-info.txt
+IDF_Commit_short=$(git -C "$IDF_PATH" rev-parse --short HEAD || echo "")
+AR_Commit_short=$(git -C "$AR_COMPS/arduino" rev-parse --short HEAD || echo "")
+echo "Framework built from
+- $IDF_REPO branch [$IDF_BRANCH](https://github.com/$IDF_REPO/tree/$IDF_BRANCH) commit [$IDF_Commit_short](https://github.com/$IDF_REPO/commits/$IDF_BRANCH/#:~:text=$IDF_Commit_short)
+- $AR_REPO branch [$AR_BRANCH](https://github.com/$AR_REPO/tree/$AR_BRANCH) commit [$AR_Commit_short](https://github.com/$AR_REPO/commits/$AR_BRANCH/#:~:text=$AR_Commit_short)
+- Arduino lib builder branch: $GIT_BRANCH" >> release-info.txt
+
 #targets_count=`jq -c '.targets[] | length' configs/builds.json`
 for target_json in `jq -c '.targets[]' configs/builds.json`; do
     target=$(echo "$target_json" | jq -c '.target' | tr -d '"')
@@ -210,7 +204,7 @@ for target_json in `jq -c '.targets[]' configs/builds.json`; do
     echo "* Target: $target"
 
     # Build Main Configs List
-    main_configs="configs/defconfig.common;configs/defconfig.$target;configs/defconfig.debug_$BUILD_DEBUG"
+    main_configs="configs/defconfig.common;configs/defconfig.$target"
     for defconf in `echo "$target_json" | jq -c '.features[]' | tr -d '"'`; do
         main_configs="$main_configs;configs/defconfig.$defconf"
     done
@@ -226,19 +220,6 @@ for target_json in `jq -c '.targets[]' configs/builds.json`; do
     idf.py -DIDF_TARGET="$target" -DSDKCONFIG_DEFAULTS="$idf_libs_configs" idf-libs
     if [ $? -ne 0 ]; then exit 1; fi
 
-    if [ "$target" == "esp32s3" ]; then
-        idf.py -DIDF_TARGET="$target" -DSDKCONFIG_DEFAULTS="$idf_libs_configs" srmodels_bin
-        if [ $? -ne 0 ]; then exit 1; fi
-        AR_SDK="$AR_TOOLS/esp32-arduino-libs/$target"
-        # sr model.bin
-        if [ -f "build/srmodels/srmodels.bin" ]; then
-            echo "$AR_SDK/esp_sr"
-            mkdir -p "$AR_SDK/esp_sr"
-            cp -f "build/srmodels/srmodels.bin" "$AR_SDK/esp_sr/"
-            cp -f "partitions.csv" "$AR_SDK/esp_sr/"
-        fi
-    fi
-
     # Build Bootloaders
     for boot_conf in `echo "$target_json" | jq -c '.bootloaders[]'`; do
         bootloader_configs="$main_configs"
@@ -297,42 +278,33 @@ for component in `ls "$AR_MANAGED_COMPS"`; do
     fi
 done
 
-# update package_esp32_index.template.json
-if [ "$BUILD_TYPE" = "all" ]; then
-    echo "* Generating package_esp32_index.template.json..."
-    python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -j "$AR_COMPS/arduino/package/package_esp32_index.template.json" -o "$AR_OUT/"
-    python3 ./tools/gen_tools_json.py -i "$IDF_PATH" -o "$TOOLS_JSON_OUT/"
-    if [ $? -ne 0 ]; then exit 1; fi
-fi
+export IDF_COMMIT=$(git -C "$IDF_PATH" rev-parse --short HEAD)
 
-# Generate PlatformIO manifest file
+# Generate PlatformIO library manifest file
 if [ "$BUILD_TYPE" = "all" ]; then
-    echo "* Generating PlatformIO manifest file..."
-    pushd $IDF_PATH
-    ibr=$(git describe --all 2>/dev/null)
-    ic=$(git -C "$IDF_PATH" rev-parse --short HEAD)
-    popd
-    python3 ./tools/gen_platformio_manifest.py -o "$TOOLS_JSON_OUT/" -s "$ibr" -c "$ic"
+    python3 ./tools/gen_pio_lib_manifest.py -o "$TOOLS_JSON_OUT/" -s "v$IDF_VERSION" -c "$IDF_COMMIT"
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
-# copy everything to arduino-esp32 installation
-if [ $COPY_OUT -eq 1 ] && [ -d "$ESP32_ARDUINO" ]; then
-    echo "* Copying to Arduino..."
-    ./tools/copy-to-arduino.sh
-    if [ $? -ne 0 ]; then exit 1; fi
-fi
+AR_VERSION=$(jq -c '.version' "$AR_COMPS/arduino/package.json" | tr -d '"')
+AR_VERSION_UNDERSCORE=`echo "$AR_VERSION" | tr . _`
 
-# push changes to esp32-arduino-libs and create pull request into arduino-esp32
-if [ $DEPLOY_OUT -eq 1 ]; then
-    echo "* Pushing to Arduino..."
-    ./tools/push-to-arduino.sh
+# Generate PlatformIO framework manifest file
+rm -rf "$AR_ROOT/package.json"
+if [ "$BUILD_TYPE" = "all" ]; then
+    python3 ./tools/gen_pio_frmwk_manifest.py -o "$AR_ROOT/" -s "v$AR_VERSION" -c "$IDF_COMMIT"
     if [ $? -ne 0 ]; then exit 1; fi
 fi
 
+# Generate core_version.h
+rm -rf "$AR_ROOT/core_version.h"
+echo "#define ARDUINO_ESP32_GIT_VER 0x$AR_Commit_short
+#define ARDUINO_ESP32_GIT_DESC $AR_VERSION
+#define ARDUINO_ESP32_RELEASE_$AR_VERSION_UNDERSCORE
+#define ARDUINO_ESP32_RELEASE \"$AR_VERSION_UNDERSCORE\"" >> "$AR_ROOT/core_version.h"
+
 # archive the build
 if [ $ARCHIVE_OUT -eq 1 ]; then
-    echo "* Archiving build..."
     ./tools/archive-build.sh "$TARGET"
     if [ $? -ne 0 ]; then exit 1; fi
 fi

tools/gen_pio_lib_manifest.py

@@ -0,0 +1,86 @@
+import argparse
+import json
+import os
+import re
+import sys
+
+MANIFEST_DATA = {
+    "name": "framework-arduinoespressif32-libs",
+    "description": "Precompiled libraries for Arduino Wiring-based Framework for the Espressif ESP32 series of SoCs",
+    "keywords": ["framework", "tasmota", "arduino", "espressif", "esp32"],
+    "license": "LGPL-2.1-or-later",
+    "repository": {
+        "type": "git",
+        "url": "https://github.com/tasmota/esp32-arduino-libs",
+    },
+}
+
+
+def convert_version(version_string):
+    """A helper function that converts a custom IDF version string
+    extracted from a Git repository to a suitable SemVer alternative. For example:
+    'release/v5.1' becomes '5.1.0',
+    'v7.7.7' becomes '7.7.7'
+    """
+
+    regex_pattern = (
+        r"v(?P<MAJOR>0|[1-9]\d*)\.(?P<MINOR>0|[1-9]\d*)\.*(?P<PATCH>0|[1-9]\d*)*"
+    )
+    match = re.search(regex_pattern, version_string)
+    if not match:
+        sys.stderr.write(
+            f"Failed to find a regex match for '{regex_pattern}' in '{version_string}'\n"
+        )
+        return ""
+
+    major, minor, patch = match.groups()
+    if not patch:
+        patch = "0"
+
+    return ".".join((major, minor, patch))
+
+
+def main(dst_dir, version_string, commit_hash):
+
+    converted_version = convert_version(version_string)
+    if not converted_version:
+        sys.stderr.write(f"Failed to convert version '{version_string}'\n")
+        return -1
+
+    manifest_file_path = os.path.join(dst_dir, "package.json")
+    with open(manifest_file_path, "w", encoding="utf8") as fp:
+        MANIFEST_DATA["version"] = f"{converted_version}+sha.{commit_hash}"
+        json.dump(MANIFEST_DATA, fp, indent=2)
+
+    print(
+        f"Generated PlatformIO libraries manifest file '{manifest_file_path}' with '{converted_version}' version"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-o",
+        "--dst-dir",
+        dest="dst_dir",
+        required=True,
+        help="Destination folder where the 'package.json' manifest will be located",
+    )
+    parser.add_argument(
+        "-s",
+        "--version-string",
+        dest="version_string",
+        required=True,
+        help="ESP-IDF version string used for compiling libraries",
+    )
+    parser.add_argument(
+        "-c",
+        "--commit-hash",
+        dest="commit_hash",
+        required=True,
+        help="ESP-IDF revision in form of a commit hash",
+    )
+    args = parser.parse_args()
+
+    sys.exit(main(args.dst_dir, args.version_string, args.commit_hash))