Merge branch 'master' into master

Author: igrr

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+----------------------------------------------------------------------------------------------------------------------------------------------------
+This entire section can be deleted if all items are checked.
+
+*By completing this PR sufficiently, you help us to improve the quality of Release Notes*
+
+### Checklist
+
+1. [ ] Please provide specific title of the PR describing the change, including the component name (eg."Update of Documentation link on Readme.md")
+2. [ ] Please provide related links (eg. Issue, other Project, submodule PR..)
+----------------------------------------------------------------------------------------------------------------------------------------------------
+
+## Summary
+Please describe your proposed PR and what it contains.
+
+## Impact
+Please describe impact of your PR and it's function.

.github/scripts/install-platformio-esp32.sh

@@ -1,19 +1,43 @@
 #!/bin/bash
 
 export PLATFORMIO_ESP32_PATH="$HOME/.platformio/packages/framework-arduinoespressif32"
-PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git#feature/idf-master"
+PLATFORMIO_ESP32_URL="https://github.com/platformio/platform-espressif32.git#feature/arduino-idf-master"
+
+XTENSA32_TOOLCHAIN_VERSION="8.4.0+2021r1"
+XTENSA32S2_TOOLCHAIN_VERSION="8.4.0+2021r1"
+RISCV_TOOLCHAIN_VERSION="8.4.0+2021r1"
+ESPTOOLPY_VERSION="~1.30100.0"
+ESPRESSIF_ORGANIZATION_NAME="espressif"
 
 echo "Installing Python Wheel ..."
 pip install wheel > /dev/null 2>&1
 
 echo "Installing PlatformIO ..."
-pip install -U https://github.com/platformio/platformio/archive/develop.zip > /dev/null 2>&1
+pip install -U https://github.com/platformio/platformio/archive/master.zip > /dev/null 2>&1
 
 echo "Installing Platform ESP32 ..."
-python -m platformio platform install $PLATFORMIO_ESP32_URL > /dev/null 2>&1
-
-echo "Replacing the framework version ..."
-python -c "import json; import os; fp=open(os.path.expanduser('~/.platformio/platforms/espressif32/platform.json'), 'r+'); data=json.load(fp); data['packages']['framework-arduinoespressif32']['version'] = '*'; fp.seek(0); fp.truncate(); json.dump(data, fp); fp.close()"
+python -m platformio platform install $PLATFORMIO_ESP32_URL  > /dev/null 2>&1
+
+echo "Replacing the package versions ..."
+replace_script="import json; import os;"
+replace_script+="fp=open(os.path.expanduser('~/.platformio/platforms/espressif32/platform.json'), 'r+');"
+replace_script+="data=json.load(fp);"
+# Use framework sources from the repository
+replace_script+="data['packages']['framework-arduinoespressif32']['version'] = '*';"
+replace_script+="del data['packages']['framework-arduinoespressif32']['owner'];"
+# Use toolchain packages from the "espressif" organization
+replace_script+="data['packages']['toolchain-xtensa-esp32']['owner']='$ESPRESSIF_ORGANIZATION_NAME';"
+replace_script+="data['packages']['toolchain-xtensa-esp32s2']['owner']='$ESPRESSIF_ORGANIZATION_NAME';"
+replace_script+="data['packages']['toolchain-riscv32-esp']['owner']='$ESPRESSIF_ORGANIZATION_NAME';"
+# Update versions to use the upstream
+replace_script+="data['packages']['toolchain-xtensa-esp32']['version']='$XTENSA32_TOOLCHAIN_VERSION';"
+replace_script+="data['packages']['toolchain-xtensa-esp32s2']['version']='$XTENSA32S2_TOOLCHAIN_VERSION';"
+replace_script+="data['packages']['toolchain-riscv32-esp']['version']='$RISCV_TOOLCHAIN_VERSION';"
+# esptool.py may require an upstream version (for now platformio is the owner)
+replace_script+="data['packages']['tool-esptoolpy']['version']='$ESPTOOLPY_VERSION';"
+# Save results
+replace_script+="fp.seek(0);fp.truncate();json.dump(data, fp, indent=2);fp.close()"
+python -c "$replace_script"
 
 if [ "$GITHUB_REPOSITORY" == "espressif/arduino-esp32" ];  then
 	echo "Linking Core..."

.github/scripts/on-pages.sh

@@ -85,53 +85,59 @@ function git_safe_upload_to_pages(){
     return $?
 }
 
-EVENT_JSON=`cat $GITHUB_EVENT_PATH`
-
-echo "GITHUB_EVENT_PATH: $GITHUB_EVENT_PATH"
-echo "EVENT_JSON: $EVENT_JSON"
-
-pages_added=`echo "$EVENT_JSON" | jq -r '.commits[].added[]'`
-echo "added: $pages_added"
-pages_modified=`echo "$EVENT_JSON" | jq -r '.commits[].modified[]'`
-echo "modified: $pages_modified"
-pages_removed=`echo "$EVENT_JSON" | jq -r '.commits[].removed[]'`
-echo "removed: $pages_removed"
-
-for page in $pages_added; do
-    if [[ $page != "README.md" && $page != "docs/"* ]]; then
-        continue
-    fi
-    echo "Adding '$page' to pages ..."
-    if [[ $page == "README.md" ]]; then
-        git_safe_upload_to_pages "index.md" "README.md"
-    else
-        git_safe_upload_to_pages "$page" "$page"
-    fi
-done
-
-for page in $pages_modified; do
-    if [[ $page != "README.md" && $page != "docs/"* ]]; then
-        continue
-    fi
-    echo "Modifying '$page' ..."
-    if [[ $page == "README.md" ]]; then
-        git_safe_upload_to_pages "index.md" "README.md"
-    else
-        git_safe_upload_to_pages "$page" "$page"
-    fi
-done
-
-for page in $pages_removed; do
-    if [[ $page != "README.md" && $page != "docs/"* ]]; then
-        continue
-    fi
-    echo "Removing '$page' from pages ..."
-    if [[ $page == "README.md" ]]; then
-        git_remove_from_pages "README.md" > /dev/null
-    else
-        git_remove_from_pages "$page" > /dev/null
-    fi
-done
+git_safe_upload_to_pages "index.md" "README.md"
+
+# At some point github stopped providing a list of edited file
+# but we also stopped havong documentation in md format,
+# so we can skip this portion safely and update just the index
+
+# EVENT_JSON=`cat $GITHUB_EVENT_PATH`
+
+# echo "GITHUB_EVENT_PATH: $GITHUB_EVENT_PATH"
+# echo "EVENT_JSON: $EVENT_JSON"
+
+# pages_added=`echo "$EVENT_JSON" | jq -r '.commits[].added[]'`
+# echo "added: $pages_added"
+# pages_modified=`echo "$EVENT_JSON" | jq -r '.commits[].modified[]'`
+# echo "modified: $pages_modified"
+# pages_removed=`echo "$EVENT_JSON" | jq -r '.commits[].removed[]'`
+# echo "removed: $pages_removed"
+
+# for page in $pages_added; do
+#     if [[ $page != "README.md" && $page != "docs/"* ]]; then
+#         continue
+#     fi
+#     echo "Adding '$page' to pages ..."
+#     if [[ $page == "README.md" ]]; then
+#         git_safe_upload_to_pages "index.md" "README.md"
+#     else
+#         git_safe_upload_to_pages "$page" "$page"
+#     fi
+# done
+
+# for page in $pages_modified; do
+#     if [[ $page != "README.md" && $page != "docs/"* ]]; then
+#         continue
+#     fi
+#     echo "Modifying '$page' ..."
+#     if [[ $page == "README.md" ]]; then
+#         git_safe_upload_to_pages "index.md" "README.md"
+#     else
+#         git_safe_upload_to_pages "$page" "$page"
+#     fi
+# done
+
+# for page in $pages_removed; do
+#     if [[ $page != "README.md" && $page != "docs/"* ]]; then
+#         continue
+#     fi
+#     echo "Removing '$page' from pages ..."
+#     if [[ $page == "README.md" ]]; then
+#         git_remove_from_pages "README.md" > /dev/null
+#     else
+#         git_remove_from_pages "$page" > /dev/null
+#     fi
+# done
 
 echo
 echo "DONE!"

.github/workflows/docs.yml

@@ -0,0 +1,38 @@
+name: ReadTheDocs CI
+
+on:
+  push:
+    branches:
+    - master
+    - release/*
+    paths:
+    - 'docs/**'
+    - '.github/workflows/docs.yml'
+  pull_request:
+    paths:
+    - 'docs/**'
+    - '.github/workflows/docs.yml'
+
+jobs:
+
+  build-docs:
+    name: Build ReadTheDocs
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Build
+      run: |
+        sudo apt update
+        sudo apt install python3-pip python3-setuptools
+        # GitHub CI installs pip3 and setuptools outside the path.
+        # Update the path to include them and run.
+        PATH=/home/runner/.local/bin:$PATH pip3 install --user -r ./docs/requirements.txt
+        cd ./docs && PATH=/home/runner/.local/bin:$PATH SPHINXOPTS="-W" make html

.github/workflows/gh-pages.yml

@@ -7,7 +7,6 @@ on:
     - pages
     paths:
     - 'README.md'
-    - 'docs/**'
     - '.github/scripts/on-pages.sh'
     - '.github/workflows/gh-pages.yml'
 

.gitignore

@@ -6,6 +6,7 @@ tools/esptool
 tools/esptool.exe
 tools/mkspiffs
 tools/mklittlefs
+tools/mkfatfs.exe
 .DS_Store
 
 #Ignore files built by Visual Studio/Visual Micro
@@ -17,4 +18,6 @@ __vm/
 .vscode/
 platform.sloeber.txt
 boards.sloeber.txt
-tools/mklittlefs
+
+# Ignore docs build (Sphinx)
+docs/build

CMakeLists.txt

@@ -34,6 +34,9 @@ set(CORE_SRCS
   cores/esp32/StreamString.cpp
   cores/esp32/USB.cpp
   cores/esp32/USBCDC.cpp
+  cores/esp32/USBMSC.cpp
+  cores/esp32/FirmwareMSC.cpp
+  cores/esp32/firmware_msc_fat.c
   cores/esp32/wiring_pulse.c
   cores/esp32/wiring_shift.c
   cores/esp32/WMath.cpp
@@ -55,7 +58,7 @@ set(LIBRARY_SRCS
   libraries/FS/src/vfs_api.cpp
   libraries/HTTPClient/src/HTTPClient.cpp
   libraries/HTTPUpdate/src/HTTPUpdate.cpp
-  libraries/LITTLEFS/src/LITTLEFS.cpp
+  libraries/LittleFS/src/LittleFS.cpp
   libraries/NetBIOS/src/NetBIOS.cpp
   libraries/Preferences/src/Preferences.cpp
   libraries/RainMaker/src/RMaker.cpp
@@ -73,6 +76,14 @@ set(LIBRARY_SRCS
   libraries/Ticker/src/Ticker.cpp
   libraries/Update/src/Updater.cpp
   libraries/Update/src/HttpsOTAUpdate.cpp
+  libraries/USB/src/USBHID.cpp
+  libraries/USB/src/USBHIDMouse.cpp
+  libraries/USB/src/USBHIDKeyboard.cpp
+  libraries/USB/src/USBHIDGamepad.cpp
+  libraries/USB/src/USBHIDConsumerControl.cpp
+  libraries/USB/src/USBHIDSystemControl.cpp
+  libraries/USB/src/USBHIDVendor.cpp
+  libraries/USB/src/USBVendor.cpp
   libraries/WebServer/src/WebServer.cpp
   libraries/WebServer/src/Parsing.cpp
   libraries/WebServer/src/detail/mimetable.cpp
@@ -140,7 +151,7 @@ set(includedirs
   libraries/FS/src
   libraries/HTTPClient/src
   libraries/HTTPUpdate/src
-  libraries/LITTLEFS/src
+  libraries/LittleFS/src
   libraries/NetBIOS/src
   libraries/Preferences/src
   libraries/RainMaker/src
@@ -151,6 +162,7 @@ set(includedirs
   libraries/SPI/src
   libraries/Ticker/src
   libraries/Update/src
+  libraries/USB/src
   libraries/WebServer/src
   libraries/WiFiClientSecure/src
   libraries/WiFi/src
@@ -161,25 +173,50 @@ set(includedirs
 set(srcs ${CORE_SRCS} ${LIBRARY_SRCS} ${BLE_SRCS})
 set(priv_includes cores/esp32/libb64)
 set(requires spi_flash mbedtls mdns esp_adc_cal wifi_provisioning nghttp)
-set(priv_requires fatfs nvs_flash app_update spiffs bootloader_support openssl bt main)
+set(priv_requires fatfs nvs_flash app_update spiffs bootloader_support openssl bt esp_ipc)
 
+idf_component_register(INCLUDE_DIRS ${includedirs} PRIV_INCLUDE_DIRS ${priv_includes} SRCS ${srcs} REQUIRES ${requires} PRIV_REQUIRES ${priv_requires})
+
+string(TOUPPER ${CONFIG_IDF_TARGET} idf_target_caps)
+target_compile_options(${COMPONENT_TARGET} PUBLIC
+    -DARDUINO=10812
+    -DARDUINO_${idf_target_caps}_DEV
+    -DARDUINO_ARCH_ESP32
+    -DARDUINO_BOARD="${idf_target_caps}_DEV"
+    -DARDUINO_VARIANT="${CONFIG_IDF_TARGET}"
+    -DESP32)
+
+if(CONFIG_AUTOSTART_ARDUINO)
+    # in autostart mode, arduino-esp32 contains app_main() function and needs to
+    # reference setup() and loop() in the main component. If we add main
+    # component to priv_requires then we create a large circular dependency
+    # (arduino-esp32 -> main -> arduino-esp32) and can get linker errors, so
+    # instead we add setup() and loop() to the undefined symbols list so the
+    # linker will always include them.
+    #
+    # (As they are C++ symbol, we need to add the C++ mangled names.)
+    target_link_libraries(${COMPONENT_LIB} INTERFACE "-u _Z5setupv -u _Z4loopv")
+endif()
+
+# This function adds a dependency on the given component if the component is included into the build.
+function(maybe_add_component component_name)
+    idf_build_get_property(components BUILD_COMPONENTS)
+    if (${component_name} IN_LIST components)
+        idf_component_get_property(lib_name ${component_name} COMPONENT_LIB)
+        target_link_libraries(${COMPONENT_LIB} PUBLIC ${lib_name})
+    endif()
+endfunction()
+
+if(IDF_TARGET MATCHES "esp32" AND CONFIG_ESP_RMAKER_TASK_STACK)
+    maybe_add_component(esp_rainmaker)
+    maybe_add_component(qrcode)
+endif()
 if(IDF_TARGET MATCHES "esp32s2|esp32s3" AND CONFIG_TINYUSB_ENABLED)
-  list(APPEND priv_requires arduino_tinyusb)
+    maybe_add_component(arduino_tinyusb)
 endif()
 if(NOT CONFIG_ARDUINO_SELECTIVE_COMPILATION OR CONFIG_ARDUINO_SELECTIVE_ArduinoOTA)
-  list(APPEND priv_requires esp_https_ota)
+    maybe_add_component(esp_https_ota)
 endif()
 if(NOT CONFIG_ARDUINO_SELECTIVE_COMPILATION OR CONFIG_ARDUINO_SELECTIVE_LITTLEFS)
-  if(CONFIG_LITTLEFS_PAGE_SIZE)
-    list(APPEND priv_requires esp_littlefs)
-  endif()
-endif()
-
-idf_component_register(INCLUDE_DIRS ${includedirs} PRIV_INCLUDE_DIRS ${priv_includes} SRCS ${srcs} REQUIRES ${requires} PRIV_REQUIRES ${priv_requires})
-
-if(IDF_TARGET STREQUAL "esp32")
-target_compile_options(${COMPONENT_TARGET} PUBLIC -DARDUINO=10812 -DARDUINO_ESP32_DEV -DARDUINO_ARCH_ESP32 -DARDUINO_BOARD="ESP32_DEV" -DARDUINO_VARIANT="esp32" -DESP32)
-endif()
-if(IDF_TARGET STREQUAL "esp32s2")
-target_compile_options(${COMPONENT_TARGET} PUBLIC -DARDUINO=10812 -DARDUINO_ESP32S2_DEV -DARDUINO_ARCH_ESP32 -DARDUINO_BOARD="ESP32S2_DEV" -DARDUINO_VARIANT="esp32s2" -DESP32)
+    maybe_add_component(esp_littlefs)
 endif()