diff --git a/.github/workflows/build-ide.yml b/.github/workflows/build-ide.yml index a2801a60cb..75e0e0583e 100644 --- a/.github/workflows/build-ide.yml +++ b/.github/workflows/build-ide.yml @@ -11,6 +11,22 @@ permissions: jobs: # Examples are built in parallel to avoid CI total job time limitation + sanity-check: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v3 + with: + submodules: false + - uses: actions/cache@v3 + with: + path: ./tools/dist + key: ${{ runner.os }}-${{ hashFiles('package/package_esp8266com_index.template.json', 'tests/common.sh', 'tests/build.sh') }} + - name: Toolchain sanity checks + run: | + bash ./tests/sanity_check.sh build-linux: name: Linux - LwIP ${{ matrix.lwip }} (${{ matrix.chunk }}) diff --git a/README.md b/README.md index 30528c8ed0..8ee45dfcb5 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Arduino core for ESP8266 WiFi chip # Quick links -- [Latest release documentation](https://arduino-esp8266.readthedocs.io/en/3.0.2/) +- [Latest release documentation](https://arduino-esp8266.readthedocs.io/en/3.1.1/) - [Current "git version" documentation](https://arduino-esp8266.readthedocs.io/en/latest/) - [Install git version](https://arduino-esp8266.readthedocs.io/en/latest/installing.html#using-git-version) ([sources](doc/installing.rst#using-git-version)) @@ -36,7 +36,7 @@ Starting with 1.6.4, Arduino allows installation of third-party platform package #### Latest release [![Latest release](https://img.shields.io/github/release/esp8266/Arduino.svg)](https://github.com/esp8266/Arduino/releases/latest/) Boards manager link: `https://arduino.esp8266.com/stable/package_esp8266com_index.json` -Documentation: [https://arduino-esp8266.readthedocs.io/en/3.0.2/](https://arduino-esp8266.readthedocs.io/en/3.0.2/) +Documentation: [https://arduino-esp8266.readthedocs.io/en/3.1.1/](https://arduino-esp8266.readthedocs.io/en/3.1.1/) ### Using git version diff --git a/cores/esp8266/Esp.cpp b/cores/esp8266/Esp.cpp index 0109bbe943..20da390008 100644 --- a/cores/esp8266/Esp.cpp +++ b/cores/esp8266/Esp.cpp @@ -471,7 +471,7 @@ bool EspClass::checkFlashCRC() { uint32_t firstPart = (uintptr_t)&__crc_len - 0x40200000; // How many bytes to check before the 1st CRC val // Start the checksum - uint32_t crc = crc32((const void*)0x40200000, firstPart, 0xffffffff); + uint32_t crc = crc32((const void*)0x40200000, firstPart); // Pretend the 2 words of crc/len are zero to be idempotent crc = crc32(z, 8, crc); // Finish the CRC calculation over the rest of flash diff --git a/cores/esp8266/IPAddress.cpp b/cores/esp8266/IPAddress.cpp index 25660ec0b3..6f68bc56e9 100644 --- a/cores/esp8266/IPAddress.cpp +++ b/cores/esp8266/IPAddress.cpp @@ -41,24 +41,14 @@ bool IPAddress::isSet () const { } IPAddress::IPAddress(uint8_t first_octet, uint8_t second_octet, uint8_t third_octet, uint8_t fourth_octet) { - setV4(); - (*this)[0] = first_octet; - (*this)[1] = second_octet; - (*this)[2] = third_octet; - (*this)[3] = fourth_octet; -} + uint8_t addr[] { + first_octet, + second_octet, + third_octet, + fourth_octet, + }; -void IPAddress::ctor32(uint32_t address) { - setV4(); - v4() = address; -} - -IPAddress::IPAddress(const uint8_t *address) { - setV4(); - (*this)[0] = address[0]; - (*this)[1] = address[1]; - (*this)[2] = address[2]; - (*this)[3] = address[3]; + *this = &addr[0]; } bool IPAddress::fromString(const char *address) { @@ -116,8 +106,10 @@ bool IPAddress::fromString4(const char *address) { } IPAddress& IPAddress::operator=(const uint8_t *address) { - setV4(); - v4() = *reinterpret_cast(address); + uint32_t value; + memcpy_P(&value, address, sizeof(value)); + + *this = value; return *this; } @@ -128,7 +120,14 @@ IPAddress& IPAddress::operator=(uint32_t address) { } bool IPAddress::operator==(const uint8_t* addr) const { - return isV4() && v4() == *reinterpret_cast(addr); + if (!isV4()) { + return false; + } + + uint32_t value; + memcpy_P(&value, addr, sizeof(value)); + + return v4() == value; } size_t IPAddress::printTo(Print& p) const { diff --git a/cores/esp8266/IPAddress.h b/cores/esp8266/IPAddress.h index 3c4d12965c..99419b92a9 100644 --- a/cores/esp8266/IPAddress.h +++ b/cores/esp8266/IPAddress.h @@ -61,19 +61,16 @@ class IPAddress: public Printable { return reinterpret_cast(&v4()); } - void ctor32 (uint32_t); - public: - // Constructors IPAddress(); IPAddress(const IPAddress&); IPAddress(IPAddress&&); IPAddress(uint8_t first_octet, uint8_t second_octet, uint8_t third_octet, uint8_t fourth_octet); - IPAddress(uint32_t address) { ctor32(address); } - IPAddress(unsigned long address) { ctor32(address); } - IPAddress(int address) { ctor32(address); } - IPAddress(const uint8_t *address); + IPAddress(uint32_t address) { *this = address; } + IPAddress(unsigned long address) { *this = address; } + IPAddress(int address) { *this = address; } + IPAddress(const uint8_t *address) { *this = address; } bool fromString(const char *address); bool fromString(const String &address) { return fromString(address.c_str()); } @@ -88,7 +85,7 @@ class IPAddress: public Printable { operator bool () { return isSet(); } // <- both are needed // generic IPv4 wrapper to uint32-view like arduino loves to see it - const uint32_t& v4() const { return ip_2_ip4(&_ip)->addr; } // for raw_address(const) + const uint32_t& v4() const { return ip_2_ip4(&_ip)->addr; } uint32_t& v4() { return ip_2_ip4(&_ip)->addr; } bool operator==(const IPAddress& addr) const { @@ -117,11 +114,18 @@ class IPAddress: public Printable { // Overloaded index operator to allow getting and setting individual octets of the address uint8_t operator[](int index) const { - return isV4()? *(raw_address() + index): 0; + if (!isV4()) { + return 0; + } + + return ip4_addr_get_byte_val(*ip_2_ip4(&_ip), index); } + uint8_t& operator[](int index) { setV4(); - return *(raw_address() + index); + + uint8_t* ptr = reinterpret_cast(&v4()); + return *(ptr + index); } // Overloaded copy operators to allow initialisation of IPAddress objects from other types diff --git a/cores/esp8266/Schedule.cpp b/cores/esp8266/Schedule.cpp index 279e78ff6c..f6c650fcf9 100644 --- a/cores/esp8266/Schedule.cpp +++ b/cores/esp8266/Schedule.cpp @@ -17,6 +17,7 @@ */ #include +#include #include "Schedule.h" #include "PolledTimeout.h" @@ -34,6 +35,7 @@ static scheduled_fn_t* sFirst = nullptr; static scheduled_fn_t* sLast = nullptr; static scheduled_fn_t* sUnused = nullptr; static int sCount = 0; +static uint32_t recurrent_max_grain_mS = 0; typedef std::function mRecFuncT; struct recurrent_fn_t @@ -130,9 +132,39 @@ bool schedule_recurrent_function_us(const std::function& fn, } rLast = item; + // grain needs to be recomputed + recurrent_max_grain_mS = 0; + return true; } +uint32_t compute_scheduled_recurrent_grain () +{ + if (recurrent_max_grain_mS == 0) + { + if (rFirst) + { + uint32_t recurrent_max_grain_uS = rFirst->callNow.getTimeout(); + for (auto it = rFirst->mNext; it; it = it->mNext) + recurrent_max_grain_uS = std::gcd(recurrent_max_grain_uS, it->callNow.getTimeout()); + if (recurrent_max_grain_uS) + // round to the upper millis + recurrent_max_grain_mS = recurrent_max_grain_uS <= 1000? 1: (recurrent_max_grain_uS + 999) / 1000; + } + +#ifdef DEBUG_ESP_CORE + static uint32_t last_grain = 0; + if (recurrent_max_grain_mS != last_grain) + { + ::printf(":rsf %u->%u\n", last_grain, recurrent_max_grain_mS); + last_grain = recurrent_max_grain_mS; + } +#endif + } + + return recurrent_max_grain_mS; +} + void run_scheduled_functions() { // prevent scheduling of new functions during this run @@ -226,6 +258,9 @@ void run_scheduled_recurrent_functions() } delete(to_ditch); + + // grain needs to be recomputed + recurrent_max_grain_mS = 0; } else { diff --git a/cores/esp8266/Schedule.h b/cores/esp8266/Schedule.h index da86e5b7f5..362d15b5f3 100644 --- a/cores/esp8266/Schedule.h +++ b/cores/esp8266/Schedule.h @@ -39,6 +39,11 @@ // scheduled function happen more often: every yield() (vs every loop()), // and time resolution is microsecond (vs millisecond). Details are below. +// compute_scheduled_recurrent_grain() is used by delay() to give a chance to +// all recurrent functions to run per their timing requirement. + +uint32_t compute_scheduled_recurrent_grain (); + // scheduled functions called once: // // * internal queue is FIFO. diff --git a/cores/esp8266/TZ.h b/cores/esp8266/TZ.h index 78369a11ed..b0fb312ba3 100644 --- a/cores/esp8266/TZ.h +++ b/cores/esp8266/TZ.h @@ -1,7 +1,7 @@ // autogenerated from https://raw.githubusercontent.com/nayarsystems/posix_tz_db/master/zones.csv // by script /tools/TZupdate.sh -// Fri Jan 6 20:48:46 UTC 2023 +// Mon Mar 20 22:00:17 UTC 2023 // // This database is autogenerated from IANA timezone database // https://raw.githubusercontent.com/nayarsystems/posix_tz_db/master/zones.csv @@ -85,7 +85,7 @@ #define TZ_America_Asuncion PSTR("<-04>4<-03>,M10.1.0/0,M3.4.0/0") #define TZ_America_Atikokan PSTR("EST5") #define TZ_America_Bahia PSTR("<-03>3") -#define TZ_America_Bahia_Banderas PSTR("CST6CDT,M4.1.0,M10.5.0") +#define TZ_America_Bahia_Banderas PSTR("CST6") #define TZ_America_Barbados PSTR("AST4") #define TZ_America_Belem PSTR("<-03>3") #define TZ_America_Belize PSTR("CST6") @@ -100,7 +100,7 @@ #define TZ_America_Cayenne PSTR("<-03>3") #define TZ_America_Cayman PSTR("EST5") #define TZ_America_Chicago PSTR("CST6CDT,M3.2.0,M11.1.0") -#define TZ_America_Chihuahua PSTR("MST7MDT,M4.1.0,M10.5.0") +#define TZ_America_Chihuahua PSTR("CST6") #define TZ_America_Costa_Rica PSTR("CST6") #define TZ_America_Creston PSTR("MST7") #define TZ_America_Cuiaba PSTR("<-04>4") @@ -117,7 +117,7 @@ #define TZ_America_Fortaleza PSTR("<-03>3") #define TZ_America_Fort_Nelson PSTR("MST7") #define TZ_America_Glace_Bay PSTR("AST4ADT,M3.2.0,M11.1.0") -#define TZ_America_Godthab PSTR("<-03>3<-02>,M3.5.0/-2,M10.5.0/-1") +#define TZ_America_Godthab PSTR("<-02>2") #define TZ_America_Goose_Bay PSTR("AST4ADT,M3.2.0,M11.1.0") #define TZ_America_Grand_Turk PSTR("EST5EDT,M3.2.0,M11.1.0") #define TZ_America_Grenada PSTR("AST4") @@ -153,14 +153,14 @@ #define TZ_America_Marigot PSTR("AST4") #define TZ_America_Martinique PSTR("AST4") #define TZ_America_Matamoros PSTR("CST6CDT,M3.2.0,M11.1.0") -#define TZ_America_Mazatlan PSTR("MST7MDT,M4.1.0,M10.5.0") +#define TZ_America_Mazatlan PSTR("MST7") #define TZ_America_Menominee PSTR("CST6CDT,M3.2.0,M11.1.0") -#define TZ_America_Merida PSTR("CST6CDT,M4.1.0,M10.5.0") +#define TZ_America_Merida PSTR("CST6") #define TZ_America_Metlakatla PSTR("AKST9AKDT,M3.2.0,M11.1.0") -#define TZ_America_Mexico_City PSTR("CST6CDT,M4.1.0,M10.5.0") +#define TZ_America_Mexico_City PSTR("CST6") #define TZ_America_Miquelon PSTR("<-03>3<-02>,M3.2.0,M11.1.0") #define TZ_America_Moncton PSTR("AST4ADT,M3.2.0,M11.1.0") -#define TZ_America_Monterrey PSTR("CST6CDT,M4.1.0,M10.5.0") +#define TZ_America_Monterrey PSTR("CST6") #define TZ_America_Montevideo PSTR("<-03>3") #define TZ_America_Montreal PSTR("EST5EDT,M3.2.0,M11.1.0") #define TZ_America_Montserrat PSTR("AST4") @@ -172,8 +172,8 @@ #define TZ_America_North_Dakota_Beulah PSTR("CST6CDT,M3.2.0,M11.1.0") #define TZ_America_North_Dakota_Center PSTR("CST6CDT,M3.2.0,M11.1.0") #define TZ_America_North_Dakota_New_Salem PSTR("CST6CDT,M3.2.0,M11.1.0") -#define TZ_America_Nuuk PSTR("<-03>3<-02>,M3.5.0/-2,M10.5.0/-1") -#define TZ_America_Ojinaga PSTR("MST7MDT,M3.2.0,M11.1.0") +#define TZ_America_Nuuk PSTR("<-02>2") +#define TZ_America_Ojinaga PSTR("CST6CDT,M3.2.0,M11.1.0") #define TZ_America_Panama PSTR("EST5") #define TZ_America_Pangnirtung PSTR("EST5EDT,M3.2.0,M11.1.0") #define TZ_America_Paramaribo PSTR("<-03>3") @@ -227,7 +227,7 @@ #define TZ_Arctic_Longyearbyen PSTR("CET-1CEST,M3.5.0,M10.5.0/3") #define TZ_Asia_Aden PSTR("<+03>-3") #define TZ_Asia_Almaty PSTR("<+06>-6") -#define TZ_Asia_Amman PSTR("EET-2EEST,M2.5.4/24,M10.5.5/1") +#define TZ_Asia_Amman PSTR("<+03>-3") #define TZ_Asia_Anadyr PSTR("<+12>-12") #define TZ_Asia_Aqtau PSTR("<+05>-5") #define TZ_Asia_Aqtobe PSTR("<+05>-5") @@ -244,14 +244,14 @@ #define TZ_Asia_Chita PSTR("<+09>-9") #define TZ_Asia_Choibalsan PSTR("<+08>-8") #define TZ_Asia_Colombo PSTR("<+0530>-5:30") -#define TZ_Asia_Damascus PSTR("EET-2EEST,M3.5.5/0,M10.5.5/0") +#define TZ_Asia_Damascus PSTR("<+03>-3") #define TZ_Asia_Dhaka PSTR("<+06>-6") #define TZ_Asia_Dili PSTR("<+09>-9") #define TZ_Asia_Dubai PSTR("<+04>-4") #define TZ_Asia_Dushanbe PSTR("<+05>-5") #define TZ_Asia_Famagusta PSTR("EET-2EEST,M3.5.0/3,M10.5.0/4") -#define TZ_Asia_Gaza PSTR("EET-2EEST,M3.4.4/48,M10.5.5/1") -#define TZ_Asia_Hebron PSTR("EET-2EEST,M3.4.4/48,M10.5.5/1") +#define TZ_Asia_Gaza PSTR("EET-2EEST,M3.4.4/50,M10.4.4/50") +#define TZ_Asia_Hebron PSTR("EET-2EEST,M3.4.4/50,M10.4.4/50") #define TZ_Asia_Ho_Chi_Minh PSTR("<+07>-7") #define TZ_Asia_Hong_Kong PSTR("HKT-8") #define TZ_Asia_Hovd PSTR("<+07>-7") @@ -294,7 +294,7 @@ #define TZ_Asia_Taipei PSTR("CST-8") #define TZ_Asia_Tashkent PSTR("<+05>-5") #define TZ_Asia_Tbilisi PSTR("<+04>-4") -#define TZ_Asia_Tehran PSTR("<+0330>-3:30<+0430>,J79/24,J263/24") +#define TZ_Asia_Tehran PSTR("<+0330>-3:30") #define TZ_Asia_Thimphu PSTR("<+06>-6") #define TZ_Asia_Tokyo PSTR("JST-9") #define TZ_Asia_Tomsk PSTR("<+07>-7") @@ -409,7 +409,7 @@ #define TZ_Pacific_Efate PSTR("<+11>-11") #define TZ_Pacific_Enderbury PSTR("<+13>-13") #define TZ_Pacific_Fakaofo PSTR("<+13>-13") -#define TZ_Pacific_Fiji PSTR("<+12>-12<+13>,M11.2.0,M1.2.3/99") +#define TZ_Pacific_Fiji PSTR("<+12>-12") #define TZ_Pacific_Funafuti PSTR("<+12>-12") #define TZ_Pacific_Galapagos PSTR("<-06>6") #define TZ_Pacific_Gambier PSTR("<-09>9") diff --git a/cores/esp8266/Updater.cpp b/cores/esp8266/Updater.cpp index 7ec9ca6926..91f2a9e6ae 100644 --- a/cores/esp8266/Updater.cpp +++ b/cores/esp8266/Updater.cpp @@ -248,12 +248,12 @@ bool UpdaterClass::end(bool evenIfRemaining){ uint32_t sigLen = 0; #ifdef DEBUG_UPDATER - DEBUG_UPDATER.printf_P(PSTR("[Updater] expected sigLen: %lu\n"), expectedSigLen); + DEBUG_UPDATER.printf_P(PSTR("[Updater] expected sigLen: %u\n"), expectedSigLen); #endif if (expectedSigLen > 0) { ESP.flashRead(sigLenAddr, &sigLen, SigSize); #ifdef DEBUG_UPDATER - DEBUG_UPDATER.printf_P(PSTR("[Updater] sigLen from flash: %lu\n"), sigLen); + DEBUG_UPDATER.printf_P(PSTR("[Updater] sigLen from flash: %u\n"), sigLen); #endif } @@ -272,7 +272,7 @@ bool UpdaterClass::end(bool evenIfRemaining){ } binSize -= (sigLen + SigSize); #ifdef DEBUG_UPDATER - DEBUG_UPDATER.printf_P(PSTR("[Updater] Adjusted size (without the signature and sigLen): %lu\n"), binSize); + DEBUG_UPDATER.printf_P(PSTR("[Updater] Adjusted size (without the signature and sigLen): %zu\n"), binSize); #endif } diff --git a/cores/esp8266/WString.cpp b/cores/esp8266/WString.cpp index 331b2a5409..1e608c3c92 100644 --- a/cores/esp8266/WString.cpp +++ b/cores/esp8266/WString.cpp @@ -197,6 +197,15 @@ bool String::reserve(unsigned int size) { return false; } +#ifdef DEBUG_ESP_PORT +static void identifyString (const String& badOne) +{ + DEBUG_ESP_PORT.printf("[String] '%." STR(OOM_STRING_BORDER_DISPLAY) "s ... %." STR(OOM_STRING_BORDER_DISPLAY) "s': ", + badOne.c_str(), + badOne.length() > OOM_STRING_BORDER_DISPLAY? badOne.c_str() + std::max((int)badOne.length() - OOM_STRING_BORDER_DISPLAY, OOM_STRING_BORDER_DISPLAY): ""); +} +#endif + bool String::changeBuffer(unsigned int maxStrLen) { // Can we use SSO here to avoid allocation? if (maxStrLen < sizeof(sso.buff) - 1) { @@ -218,16 +227,19 @@ bool String::changeBuffer(unsigned int maxStrLen) { } // Fallthrough to normal allocator size_t newSize = (maxStrLen + 16) & (~0xf); -#ifdef DEBUG_ESP_OOM +#ifdef DEBUG_ESP_PORT if (!isSSO() && capacity() >= OOM_STRING_THRESHOLD_REALLOC_WARN && maxStrLen > capacity()) { // warn when badly re-allocating - DEBUGV("[String] Reallocating large String(%d -> %d bytes) '%." STR(OOM_STRING_BORDER_DISPLAY) "s ... %." STR(OOM_STRING_BORDER_DISPLAY) "s'\n", - len(), maxStrLen, c_str(), - len() > OOM_STRING_BORDER_DISPLAY? c_str() + std::max((int)len() - OOM_STRING_BORDER_DISPLAY, OOM_STRING_BORDER_DISPLAY): ""); + identifyString(*this); + DEBUG_ESP_PORT.printf("Reallocating large String(%d -> %d bytes)\n", len(), maxStrLen); } #endif // Make sure we can fit newsize in the buffer if (newSize > CAPACITY_MAX) { +#ifdef DEBUG_ESP_PORT + identifyString(*this); + DEBUG_ESP_PORT.printf("Maximum capacity reached (" STR(CAPACITY_MAX) ")\n"); +#endif return false; } uint16_t oldLen = len(); @@ -247,6 +259,10 @@ bool String::changeBuffer(unsigned int maxStrLen) { setBuffer(newbuffer); return true; } +#ifdef DEBUG_ESP_PORT + identifyString(*this); + DEBUG_ESP_PORT.printf("OOM: %d -> %zu bytes\n", isSSO() ? 0: capacity(), newSize); +#endif return false; } @@ -804,7 +820,7 @@ void String::replace(const String &find, const String &replace) { if (size == len()) return; if (size > capacity() && !changeBuffer(size)) - return; // XXX: tell user! + return; int index = len() - 1; while (index >= 0 && (index = lastIndexOf(find, index)) >= 0) { readFrom = wbuffer() + index + find.len(); diff --git a/cores/esp8266/core_esp8266_features.h b/cores/esp8266/core_esp8266_features.h index 43ba31674e..9c574f20c7 100644 --- a/cores/esp8266/core_esp8266_features.h +++ b/cores/esp8266/core_esp8266_features.h @@ -20,11 +20,9 @@ */ - #ifndef CORE_ESP8266_FEATURES_H #define CORE_ESP8266_FEATURES_H - #define CORE_HAS_LIBB64 #define CORE_HAS_BASE64_CLASS #define CORE_HAS_CXA_GUARD @@ -33,9 +31,10 @@ #define WIFI_HAS_EVENT_CALLBACK #define WIFI_IS_OFF_AT_BOOT -#include // malloc() +#include // bool #include // size_t #include +#include // malloc() #ifndef __STRINGIFY #define __STRINGIFY(a) #a @@ -118,6 +117,7 @@ int esp_get_cpu_freq_mhz() #else inline int esp_get_cpu_freq_mhz() { + uint8_t system_get_cpu_freq(void); return system_get_cpu_freq(); } #endif @@ -129,7 +129,7 @@ void enablePhaseLockedWaveform(void); // Determine when the sketch runs on ESP8285 #if !defined(CORE_MOCK) -bool __attribute__((const, nothrow)) esp_is_8285(); +bool esp_is_8285() __attribute__((const, nothrow)); #else inline bool esp_is_8285() { diff --git a/cores/esp8266/core_esp8266_main.cpp b/cores/esp8266/core_esp8266_main.cpp index b73d3d3a89..ccacc95e01 100644 --- a/cores/esp8266/core_esp8266_main.cpp +++ b/cores/esp8266/core_esp8266_main.cpp @@ -22,6 +22,9 @@ //This may be used to change user task stack size: //#define CONT_STACKSIZE 4096 + +#include + #include #include "Schedule.h" extern "C" { @@ -165,10 +168,18 @@ extern "C" void esp_delay(unsigned long ms) __attribute__((weak, alias("__esp_de bool esp_try_delay(const uint32_t start_ms, const uint32_t timeout_ms, const uint32_t intvl_ms) { uint32_t expired = millis() - start_ms; if (expired >= timeout_ms) { - return true; + return true; // expired } - esp_delay(std::min((timeout_ms - expired), intvl_ms)); - return false; + + // compute greatest chunked delay with respect to scheduled recurrent functions + uint32_t grain_ms = std::gcd(intvl_ms, compute_scheduled_recurrent_grain()); + + // recurrent scheduled functions will be called from esp_delay()->esp_suspend() + esp_delay(grain_ms > 0 ? + std::min((timeout_ms - expired), grain_ms): + (timeout_ms - expired)); + + return false; // expiration must be checked again } extern "C" void __yield() { diff --git a/cores/esp8266/core_esp8266_phy.cpp b/cores/esp8266/core_esp8266_phy.cpp index 4a32b44810..2a324a85f9 100644 --- a/cores/esp8266/core_esp8266_phy.cpp +++ b/cores/esp8266/core_esp8266_phy.cpp @@ -360,10 +360,7 @@ void user_rf_pre_init() { // *((volatile uint32_t*) 0x60000710) = 0; spoof_init_data = false; - volatile uint32_t* rtc_reg = (volatile uint32_t*) 0x60001000; - rtc_reg[30] = 0; - system_set_os_print(0); int rf_mode = __get_rf_mode(); if (rf_mode >= 0) { system_phy_set_rfoption(rf_mode); diff --git a/cores/esp8266/core_esp8266_postmortem.cpp b/cores/esp8266/core_esp8266_postmortem.cpp index bd69748815..c367deeced 100644 --- a/cores/esp8266/core_esp8266_postmortem.cpp +++ b/cores/esp8266/core_esp8266_postmortem.cpp @@ -110,10 +110,26 @@ static void cut_here() { ets_putc('\n'); } -void __wrap_system_restart_local() { - register uint32_t sp asm("a1"); - uint32_t sp_dump = sp; - +/* + Add some assembly to grab the stack pointer and pass it as an argument before + it grows for the target function. Should stabilize the stack offsets, used to + find the relevant stack content for dumping. +*/ +extern "C" void __wrap_system_restart_local(void); +asm( + ".section .text.__wrap_system_restart_local,\"ax\",@progbits\n\t" + ".literal_position\n\t" + ".align 4\n\t" + ".global __wrap_system_restart_local\n\t" + ".type __wrap_system_restart_local, @function\n\t" + "\n" +"__wrap_system_restart_local:\n\t" + "mov a2, a1\n\t" + "j postmortem_report\n\t" + ".size __wrap_system_restart_local, .-__wrap_system_restart_local\n\t" +); + +static void postmortem_report(uint32_t sp_dump) { struct rst_info rst_info; memset(&rst_info, 0, sizeof(rst_info)); if (s_user_reset_reason == REASON_DEFAULT_RST) @@ -152,12 +168,22 @@ void __wrap_system_restart_local() { else if (rst_info.reason == REASON_EXCEPTION_RST) { // The GCC divide routine in ROM jumps to the address below and executes ILL (00 00 00) on div-by-zero // In that case, print the exception as (6) which is IntegerDivZero - bool div_zero = (rst_info.exccause == 0) && (rst_info.epc1 == 0x4000dce5); + uint32_t epc1 = rst_info.epc1; + uint32_t exccause = rst_info.exccause; + bool div_zero = (exccause == 0) && (epc1 == 0x4000dce5u); + if (div_zero) { + exccause = 6; + // In place of the detached 'ILL' instruction., redirect attention + // back to the code that called the ROM divide function. + __asm__ __volatile__("rsr.excsave1 %0\n\t" : "=r"(epc1) :: "memory"); + } ets_printf_P(PSTR("\nException (%d):\nepc1=0x%08x epc2=0x%08x epc3=0x%08x excvaddr=0x%08x depc=0x%08x\n"), - div_zero ? 6 : rst_info.exccause, rst_info.epc1, rst_info.epc2, rst_info.epc3, rst_info.excvaddr, rst_info.depc); + exccause, epc1, rst_info.epc2, rst_info.epc3, rst_info.excvaddr, rst_info.depc); } else if (rst_info.reason == REASON_SOFT_WDT_RST) { ets_printf_P(PSTR("\nSoft WDT reset\n")); + ets_printf_P(PSTR("\nException (%d):\nepc1=0x%08x epc2=0x%08x epc3=0x%08x excvaddr=0x%08x depc=0x%08x\n"), + rst_info.exccause, /* Address executing at time of Soft WDT level-1 interrupt */ rst_info.epc1, 0, 0, 0, 0); } else if (rst_info.reason == REASON_USER_STACK_SMASH) { ets_printf_P(PSTR("\nStack smashing detected.\n")); @@ -174,16 +200,31 @@ void __wrap_system_restart_local() { // amount of stack taken by interrupt or exception handler // and everything up to __wrap_system_restart_local - // (determined empirically, might break) + // ~(determined empirically, might break)~ uint32_t offset = 0; if (rst_info.reason == REASON_SOFT_WDT_RST) { - offset = 0x1a0; + // Stack Tally + // 256 User Exception vector handler reserves stack space + // directed to _xtos_l1int_handler function in Boot ROM + // 48 wDev_ProcessFiq - its address appears in a vector table at 0x3FFFC27C + // 16 ?unnamed? - index into a table, pull out pointer, and call if non-zero + // appears near near wDev_ProcessFiq + // 32 pp_soft_wdt_feed_local - gather the specifics and call __wrap_system_restart_local + offset = 32 + 16 + 48 + 256; } else if (rst_info.reason == REASON_EXCEPTION_RST) { - offset = 0x190; + // Stack Tally + // 256 Exception vector reserves stack space + // filled in by "C" wrapper handler + // 16 Handler level 1 - enable icache + // 64 Handler level 2 - exception report + offset = 64 + 16 + 256; } else if (rst_info.reason == REASON_WDT_RST) { - offset = 0x10; + offset = 16; + } + else if (rst_info.reason == REASON_USER_SWEXCEPTION_RST) { + offset = 16; } ets_printf_P(PSTR("\n>>>stack>>>\n")); @@ -280,8 +321,9 @@ static void raise_exception() { s_user_reset_reason = REASON_USER_SWEXCEPTION_RST; ets_printf_P(PSTR("\nUser exception (panic/abort/assert)")); - __wrap_system_restart_local(); - + uint32_t sp; + __asm__ __volatile__ ("mov %0, a1\n\t" : "=r"(sp) :: "memory"); + postmortem_report(sp); while (1); // never reached, needed to satisfy "noreturn" attribute } @@ -321,7 +363,9 @@ void __stack_chk_fail(void) { if (gdb_present()) __asm__ __volatile__ ("syscall"); // triggers GDB when enabled - __wrap_system_restart_local(); + uint32_t sp; + __asm__ __volatile__ ("mov %0, a1\n\t" : "=r"(sp) :: "memory"); + postmortem_report(sp); __builtin_unreachable(); // never reached, needed to satisfy "noreturn" attribute } diff --git a/cores/esp8266/core_esp8266_wiring.cpp b/cores/esp8266/core_esp8266_wiring.cpp index 40a996d560..3054d39338 100644 --- a/cores/esp8266/core_esp8266_wiring.cpp +++ b/cores/esp8266/core_esp8266_wiring.cpp @@ -34,7 +34,9 @@ static uint32_t micros_overflow_count = 0; #define REPEAT 1 void __delay(unsigned long ms) { - esp_delay(ms); + // Use API letting recurrent scheduled functions run in background + // but stay blocked in delay until ms is expired. + esp_delay(ms, [](){ return true; }); } void delay(unsigned long ms) __attribute__ ((weak, alias("__delay"))); diff --git a/cores/esp8266/coredecls.h b/cores/esp8266/coredecls.h index 1e2776d71b..73af6d8cf1 100644 --- a/cores/esp8266/coredecls.h +++ b/cores/esp8266/coredecls.h @@ -1,6 +1,5 @@ -#ifndef __COREDECLS_H -#define __COREDECLS_H +#pragma once #include "core_esp8266_features.h" @@ -20,18 +19,20 @@ void esp_delay(unsigned long ms); void esp_schedule(); void esp_yield(); void tune_timeshift64 (uint64_t now_us); +bool sntp_set_timezone_in_seconds(int32_t timezone); + void disable_extra4k_at_link_time (void) __attribute__((noinline)); void enable_wifi_enterprise_patch(void) __attribute__((noinline)); -bool sntp_set_timezone_in_seconds(int32_t timezone); void __disableWiFiAtBootTime (void) __attribute__((noinline)); void __real_system_restart_local() __attribute__((noreturn)); -uint32_t sqrt32 (uint32_t n); -uint32_t crc32 (const void* data, size_t length, uint32_t crc = 0xffffffff); +uint32_t sqrt32(uint32_t n); #ifdef __cplusplus } +uint32_t crc32(const void* data, size_t length, uint32_t crc = 0xffffffff); + #include using BoolCB = std::function; @@ -53,14 +54,15 @@ inline void esp_suspend(T&& blocked) { // Try to delay until timeout_ms has expired since start_ms. // Returns true if timeout_ms has completely expired on entry. // Otherwise returns false after delaying for the relative -// remainder of timeout_ms, or an absolute intvl_ms, whichever is shorter. +// remainder of timeout_ms, or an absolute intvl_ms, whichever is shorter +// and possibly amended by recurrent scheduled functions timing grain. // The delay may be asynchronously cancelled, before that timeout is reached. bool esp_try_delay(const uint32_t start_ms, const uint32_t timeout_ms, const uint32_t intvl_ms); // This overload of esp_delay() delays for a duration of at most timeout_ms milliseconds. -// Whenever it is resumed, as well as every intvl_ms millisconds, it performs -// the blocked callback, and if that returns true, it keeps delaying for the remainder -// of the original timeout_ms period. +// Whenever it is resumed, as well as at most every intvl_ms millisconds and depending on +// recurrent scheduled functions, it performs the blocked callback, and if that returns true, +// it keeps delaying for the remainder of the original timeout_ms period. template inline void esp_delay(const uint32_t timeout_ms, T&& blocked, const uint32_t intvl_ms) { const auto start_ms = millis(); @@ -77,5 +79,3 @@ inline void esp_delay(const uint32_t timeout_ms, T&& blocked) { } #endif // __cplusplus - -#endif // __COREDECLS_H diff --git a/cores/esp8266/crc32.cpp b/cores/esp8266/crc32.cpp index c2fdf928e7..42de0c1b91 100644 --- a/cores/esp8266/crc32.cpp +++ b/cores/esp8266/crc32.cpp @@ -23,7 +23,7 @@ #include "pgmspace.h" // moved from core_esp8266_eboot_command.cpp -uint32_t crc32 (const void* data, size_t length, uint32_t crc /*= 0xffffffff*/) +uint32_t crc32 (const void* data, size_t length, uint32_t crc) { const uint8_t* ldata = (const uint8_t*)data; while (length--) diff --git a/cores/esp8266/esp_priv.h b/cores/esp8266/esp_priv.h index 305197e1c9..6e11208bd1 100644 --- a/cores/esp8266/esp_priv.h +++ b/cores/esp8266/esp_priv.h @@ -23,9 +23,8 @@ #if defined(CORE_MOCK) -constexpr bool __byteAddressable(const void* addr) +constexpr bool __byteAddressable(const void*) { - (void)addr; return true; } @@ -34,7 +33,7 @@ constexpr bool __byteAddressable(const void* addr) #include // returns true when addr can be used without "pgm_" functions or non32xfer service -constexpr bool __byteAddressable(const void* addr) +inline bool __byteAddressable(const void* addr) { return addr < (const void*)(XCHAL_DATARAM0_VADDR + XCHAL_DATARAM0_SIZE); } diff --git a/cores/esp8266/flash_hal.h b/cores/esp8266/flash_hal.h index 682e1dcfb1..061b1d0b08 100644 --- a/cores/esp8266/flash_hal.h +++ b/cores/esp8266/flash_hal.h @@ -43,7 +43,7 @@ extern const flash_map_s __flashdesc[]; #define FLASH_MAP_SETUP_CONFIG(conf) FLASH_MAP_SETUP_CONFIG_ATTR(,conf) #define FLASH_MAP_SETUP_CONFIG_ATTR(attr, conf...) \ - const flash_map_s __flashdesc[] PROGMEM = conf; \ + extern const flash_map_s __flashdesc[] PROGMEM attr = conf; \ const char *flashinit (void) attr; \ const char *flashinit (void) \ { \ diff --git a/cores/esp8266/wpa2_eap_patch.cpp b/cores/esp8266/wpa2_eap_patch.cpp index 346a748b1d..268c65e7b2 100644 --- a/cores/esp8266/wpa2_eap_patch.cpp +++ b/cores/esp8266/wpa2_eap_patch.cpp @@ -13,7 +13,7 @@ #if defined(NONOSDK22x_190703) || \ defined(NONOSDK22x_191122) || \ defined(NONOSDK22x_191105) || \ - defined(NONOSDK22x_191124) || \ + defined(NONOSDK22x_191024) || \ defined(NONOSDK22x_190313) || \ defined(NONOSDK221) || \ defined(NONOSDK305) diff --git a/doc/gdb.rst b/doc/gdb.rst index 53e9f07fbb..79e221e844 100644 --- a/doc/gdb.rst +++ b/doc/gdb.rst @@ -380,4 +380,33 @@ breakpoint) command in GDB while debugging instead of the more common ``break`` command, since ``thb`` will remove the breakpoint once it is reached automatically and save you some trouble. +Because of the single hardware breakpoint limitation, you must pay careful +attention to the output from ``gdb`` when you set a breakpoint. If your +breakpoint expression matches multiple locations, as in this example: +.. code:: bash + + (gdb) break loop + Breakpoint 1 at 0x40202c84: loop. (2 locations) + +Then you will be unable to ``continue``: + +.. code:: bash + + (gdb) cont + Continuing. + Note: automatically using hardware breakpoints for read-only addresses. + Warning: + Cannot insert hardware breakpoint 1. + Could not insert hardware breakpoints: + You may have requested too many hardware breakpoints/watchpoints. + +You can resolve this situation by deleting the previous breakpoint and +using a more specific breakpoint expression: + +.. code:: bash + + (gdb) delete + Delete all breakpoints? (y or n) y + (gdb) break mysketch.ino:loop + Breakpoint 2 at 0x40202c84: file .../mysketch.ino, line 113. diff --git a/doc/installing.rst b/doc/installing.rst index 8f04b90f05..9a9830a11e 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -44,7 +44,7 @@ Prerequisites - Arduino 1.6.8 (or newer, current working version is 1.8.5) - git -- Python 3.x (https://python.org) +- Python ≥3.7 (https://python.org) - terminal, console, or command prompt (depending on your OS) - Internet connection - Uninstalling any core version installed via Board Manager diff --git a/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino b/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino index 8d4f9c3015..4dc96622ba 100644 --- a/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino +++ b/libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino @@ -67,7 +67,7 @@ void loop() { http.end(); } else { - Serial.printf("[HTTP} Unable to connect\n"); + Serial.println("[HTTP] Unable to connect"); } } diff --git a/libraries/ESP8266HTTPClient/examples/BasicHttpsClient/BasicHttpsClient.ino b/libraries/ESP8266HTTPClient/examples/BasicHttpsClient/BasicHttpsClient.ino index b8c4c5b328..bcb69d9641 100644 --- a/libraries/ESP8266HTTPClient/examples/BasicHttpsClient/BasicHttpsClient.ino +++ b/libraries/ESP8266HTTPClient/examples/BasicHttpsClient/BasicHttpsClient.ino @@ -9,12 +9,15 @@ #include #include - #include - #include -// Fingerprint for demo URL, expires on June 2, 2021, needs to be updated well before this date -const uint8_t fingerprint[20] = { 0x40, 0xaf, 0x00, 0x6b, 0xec, 0x90, 0x22, 0x41, 0x8e, 0xa3, 0xad, 0xfa, 0x1a, 0xe8, 0x25, 0x41, 0x1d, 0x1a, 0x54, 0xb3 }; + +#include "certs.h" + +#ifndef STASSID +#define STASSID "your-ssid" +#define STAPSK "your-password" +#endif ESP8266WiFiMulti WiFiMulti; @@ -27,14 +30,9 @@ void setup() { Serial.println(); Serial.println(); - for (uint8_t t = 4; t > 0; t--) { - Serial.printf("[SETUP] WAIT %d...\n", t); - Serial.flush(); - delay(1000); - } - WiFi.mode(WIFI_STA); - WiFiMulti.addAP("SSID", "PASSWORD"); + WiFiMulti.addAP(STASSID, STAPSK); + Serial.println("setup() done connecting to ssid '" STASSID "'"); } void loop() { @@ -43,14 +41,14 @@ void loop() { std::unique_ptr client(new BearSSL::WiFiClientSecure); - client->setFingerprint(fingerprint); + client->setFingerprint(fingerprint_sni_cloudflaressl_com); // Or, if you happy to ignore the SSL certificate, then use the following line instead: // client->setInsecure(); HTTPClient https; Serial.print("[HTTPS] begin...\n"); - if (https.begin(*client, "https://jigsaw.w3.org/HTTP/connection.html")) { // HTTPS + if (https.begin(*client, jigsaw_host, jigsaw_port)) { // HTTPS Serial.print("[HTTPS] GET...\n"); // start connection and send HTTP header diff --git a/libraries/ESP8266HTTPClient/examples/BasicHttpsClient/certUpdate b/libraries/ESP8266HTTPClient/examples/BasicHttpsClient/certUpdate new file mode 100755 index 0000000000..71b036dc85 --- /dev/null +++ b/libraries/ESP8266HTTPClient/examples/BasicHttpsClient/certUpdate @@ -0,0 +1,2 @@ +cd ${0%/*} 2>/dev/null +python3 ../../../../tools/cert.py -s jigsaw.w3.org -n jigsaw > certs.h diff --git a/libraries/ESP8266HTTPClient/examples/BasicHttpsClient/certs.h b/libraries/ESP8266HTTPClient/examples/BasicHttpsClient/certs.h new file mode 100644 index 0000000000..0263070421 --- /dev/null +++ b/libraries/ESP8266HTTPClient/examples/BasicHttpsClient/certs.h @@ -0,0 +1,58 @@ + +// this file is autogenerated - any modification will be overwritten +// unused symbols will not be linked in the final binary +// generated on 2023-03-20 23:02:42 +// by ['../../../../tools/cert.py', '-s', 'jigsaw.w3.org', '-n', 'jigsaw'] + +#pragma once + +//////////////////////////////////////////////////////////// +// certificate chain for jigsaw.w3.org:443 + +const char* jigsaw_host = "jigsaw.w3.org"; +const uint16_t jigsaw_port = 443; + +// CN: sni.cloudflaressl.com => name: sni_cloudflaressl_com +// not valid before: 2023-02-14 00:00:00 +// not valid after: 2024-02-14 23:59:59 +const char fingerprint_sni_cloudflaressl_com [] PROGMEM = "70:7c:82:07:f3:58:18:87:25:42:31:83:45:86:bd:17:86:71:4e:1f"; +const char pubkey_sni_cloudflaressl_com [] PROGMEM = R"PUBKEY( +-----BEGIN PUBLIC KEY----- +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE/NU/7vfdymScyhfx81ieO8XiwGqq +TU4tjeWzSosWSpmQwnGmRqiU2h2wyT9uYxRme6uQ0yLedf4nz9ks+4OxtA== +-----END PUBLIC KEY----- +)PUBKEY"; + +// http://cacerts.digicert.com/CloudflareIncECCCA-3.crt +// CN: Cloudflare Inc ECC CA-3 => name: Cloudflare_Inc_ECC_CA_3 +// not valid before: 2020-01-27 12:48:08 +// not valid after: 2024-12-31 23:59:59 +const char cert_Cloudflare_Inc_ECC_CA_3 [] PROGMEM = R"CERT( +-----BEGIN CERTIFICATE----- +MIIDzTCCArWgAwIBAgIQCjeHZF5ftIwiTv0b7RQMPDANBgkqhkiG9w0BAQsFADBa +MQswCQYDVQQGEwJJRTESMBAGA1UEChMJQmFsdGltb3JlMRMwEQYDVQQLEwpDeWJl +clRydXN0MSIwIAYDVQQDExlCYWx0aW1vcmUgQ3liZXJUcnVzdCBSb290MB4XDTIw +MDEyNzEyNDgwOFoXDTI0MTIzMTIzNTk1OVowSjELMAkGA1UEBhMCVVMxGTAXBgNV +BAoTEENsb3VkZmxhcmUsIEluYy4xIDAeBgNVBAMTF0Nsb3VkZmxhcmUgSW5jIEVD +QyBDQS0zMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEua1NZpkUC0bsH4HRKlAe +nQMVLzQSfS2WuIg4m4Vfj7+7Te9hRsTJc9QkT+DuHM5ss1FxL2ruTAUJd9NyYqSb +16OCAWgwggFkMB0GA1UdDgQWBBSlzjfq67B1DpRniLRF+tkkEIeWHzAfBgNVHSME +GDAWgBTlnVkwgkdYzKz6CFQ2hns6tQRN8DAOBgNVHQ8BAf8EBAMCAYYwHQYDVR0l +BBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMBIGA1UdEwEB/wQIMAYBAf8CAQAwNAYI +KwYBBQUHAQEEKDAmMCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2VydC5j +b20wOgYDVR0fBDMwMTAvoC2gK4YpaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL09t +bmlyb290MjAyNS5jcmwwbQYDVR0gBGYwZDA3BglghkgBhv1sAQEwKjAoBggrBgEF +BQcCARYcaHR0cHM6Ly93d3cuZGlnaWNlcnQuY29tL0NQUzALBglghkgBhv1sAQIw +CAYGZ4EMAQIBMAgGBmeBDAECAjAIBgZngQwBAgMwDQYJKoZIhvcNAQELBQADggEB +AAUkHd0bsCrrmNaF4zlNXmtXnYJX/OvoMaJXkGUFvhZEOFp3ArnPEELG4ZKk40Un ++ABHLGioVplTVI+tnkDB0A+21w0LOEhsUCxJkAZbZB2LzEgwLt4I4ptJIsCSDBFe +lpKU1fwg3FZs5ZKTv3ocwDfjhUkV+ivhdDkYD7fa86JXWGBPzI6UAPxGezQxPk1H +goE6y/SJXQ7vTQ1unBuCJN0yJV0ReFEQPaA1IwQvZW+cwdFD19Ae8zFnWSfda9J1 +CZMRJCQUzym+5iPDuI9yP+kHyCREU3qzuWFloUwOxkgAyXVjBYdwRVKD05WdRerw +6DEdfgkfCv4+3ao8XnTSrLE= +-----END CERTIFICATE----- +)CERT"; + +// end of certificate chain for jigsaw.w3.org:443 +//////////////////////////////////////////////////////////// + diff --git a/libraries/ESP8266SdFat b/libraries/ESP8266SdFat index 43a65b42d5..46151c90a4 160000 --- a/libraries/ESP8266SdFat +++ b/libraries/ESP8266SdFat @@ -1 +1 @@ -Subproject commit 43a65b42d5a827092188cfe11fb27237da252692 +Subproject commit 46151c90a410a6f983f2f8c147e13086aaecdd8e diff --git a/libraries/ESP8266WebServer/examples/AdvancedWebServer/AdvancedWebServer.ino b/libraries/ESP8266WebServer/examples/AdvancedWebServer/AdvancedWebServer.ino index 36511917db..b96f977373 100644 --- a/libraries/ESP8266WebServer/examples/AdvancedWebServer/AdvancedWebServer.ino +++ b/libraries/ESP8266WebServer/examples/AdvancedWebServer/AdvancedWebServer.ino @@ -32,6 +32,7 @@ #include #include #include +#include #ifndef STASSID #define STASSID "your-ssid" @@ -47,14 +48,14 @@ const int led = 13; void handleRoot() { digitalWrite(led, 1); - char temp[400]; int sec = millis() / 1000; int min = sec / 60; int hr = min / 60; - snprintf(temp, 400, - - "\ + StreamString temp; + temp.reserve(500); // Preallocate a large chunk to avoid memory fragmentation + temp.printf("\ +\ \ \ ESP8266 Demo\ @@ -68,9 +69,8 @@ void handleRoot() { \ \ ", - - hr, min % 60, sec % 60); - server.send(200, "text/html", temp); + hr, min % 60, sec % 60); + server.send(200, "text/html", temp.c_str()); digitalWrite(led, 0); } diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 6ffdbbd3b6..eca341aee5 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -296,12 +296,35 @@ void ESP8266WebServerTemplate::handleClient() { bool keepCurrentClient = false; bool callYield = false; - DBGWS("http-server loop: conn=%d avail=%d status=%s\n", - _currentClient.connected(), _currentClient.available(), - _currentStatus==HC_NONE?"none": - _currentStatus==HC_WAIT_READ?"wait-read": - _currentStatus==HC_WAIT_CLOSE?"wait-close": - "??"); +#ifdef DEBUG_ESP_HTTP_SERVER + + struct compare_s + { + uint8_t connected; + int available; + HTTPClientStatus status; + bool operator != (const compare_s& o) + { + return o.connected != connected + || o.available != available + || o.status != status; + } + }; + static compare_s last { false, 0, HC_NONE }; + compare_s now { _currentClient.connected(), _currentClient.available(), _currentStatus }; + + if (last != now) + { + DBGWS("http-server loop: conn=%d avail=%d status=%s\n", + _currentClient.connected(), _currentClient.available(), + _currentStatus==HC_NONE?"none": + _currentStatus==HC_WAIT_READ?"wait-read": + _currentStatus==HC_WAIT_CLOSE?"wait-close": + "??"); + last = now; + } + +#endif // DEBUG_ESP_HTTP_SERVER if (_currentClient.connected() || _currentClient.available()) { if (_currentClient.available() && _keepAlive) { diff --git a/libraries/ESP8266WiFi/examples/BearSSL_Validation/certs.h b/libraries/ESP8266WiFi/examples/BearSSL_Validation/certs.h index 5a96edfad2..74f244c4eb 100644 --- a/libraries/ESP8266WiFi/examples/BearSSL_Validation/certs.h +++ b/libraries/ESP8266WiFi/examples/BearSSL_Validation/certs.h @@ -1,7 +1,7 @@ // this file is autogenerated - any modification will be overwritten // unused symbols will not be linked in the final binary -// generated on 2023-01-06 21:52:52 +// generated on 2023-03-20 23:02:42 // by ['../../../../tools/cert.py', '-s', 'www.example.com', '-n', 'SSL'] #pragma once @@ -13,18 +13,18 @@ const char* SSL_host = "www.example.com"; const uint16_t SSL_port = 443; // CN: www.example.org => name: www_example_org -// not valid before: 2022-03-14 00:00:00 -// not valid after: 2023-03-14 23:59:59 -const char fingerprint_www_example_org [] PROGMEM = "df:81:df:a6:b6:1e:af:df:ff:fe:1a:25:02:40:db:5d:2e:6c:ee:25"; +// not valid before: 2023-01-13 00:00:00 +// not valid after: 2024-02-13 23:59:59 +const char fingerprint_www_example_org [] PROGMEM = "f2:aa:d7:3d:32:68:3b:71:6d:2a:7d:61:b5:1c:6d:57:64:ab:38:99"; const char pubkey_www_example_org [] PROGMEM = R"PUBKEY( -----BEGIN PUBLIC KEY----- -MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlV2WY5rlGn1fpwvuBhj0 -nVBcNxCxkHUG/pJG4HvaJen7YIZ1mLc7/P4snOJZiEfwWFTikHNbcUCcYiKG8JkF -ebZOYMc1U9PiEtVWGU4kuYuxiXpD8oMPin1B0SgrF7gKfO1//I2weJdAUjgZuXBC -PAlhz2EnHddzXUtwm9XuOLO/Y6LATVMsbp8/lXnfo/bX0UgJ7C0aVqOu07A0Vr6O -kPxwWmOvF3cRKhVCM7U4B51KK+IsWRLm8cVW1IaXjwhGzW7BR6EI3sxCQ4Wnc6HV -PSgmomLWWWkIGFPAwcWUB4NC12yhCO5iW/dxNMWNLMRVtnZAyq6FpZ8wFK6j4OMw -MwIDAQAB +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwoB3iVm4RW+6StkR+nut +x1fQevu2+t0Fu6KBcbvhfyHSXy7w0nJOdTT4jWLjStpRkNQBPZwMwHH35i+21gdn +JtDe/xfO8IX9McFmyodlBUcqX8CruIzDv9AXf2OjXPBG+4aq+03XKl5/muATl32+ ++301Vw1dXoGYNeoWQqLTsHT3WS3tOOf+ehuzNuZ+rj+ephaD3lMBToEArrtC9R91 +KTTN6YSAOK48NxTA8CfOMFK5itxfIqB5+E9OSQTidXyqLyoeA+xxTKMqYfxvypEe +k1oueAhY9u67NCBdmuavxtfyvwp7+o6Sd+NsewxAhmRKFexw13KOYzDhC+9aMJcu +JQIDAQAB -----END PUBLIC KEY----- )PUBKEY"; diff --git a/libraries/ESP8266WiFi/examples/HTTPSRequest/certs.h b/libraries/ESP8266WiFi/examples/HTTPSRequest/certs.h index 49d250c269..f59ae673b3 100644 --- a/libraries/ESP8266WiFi/examples/HTTPSRequest/certs.h +++ b/libraries/ESP8266WiFi/examples/HTTPSRequest/certs.h @@ -1,7 +1,7 @@ // this file is autogenerated - any modification will be overwritten // unused symbols will not be linked in the final binary -// generated on 2023-01-06 21:52:53 +// generated on 2023-03-20 23:02:43 // by ['../../../../tools/cert.py', '-s', 'api.github.com', '-n', 'github'] #pragma once @@ -13,13 +13,13 @@ const char* github_host = "api.github.com"; const uint16_t github_port = 443; // CN: *.github.com => name: __github_com -// not valid before: 2022-03-16 00:00:00 -// not valid after: 2023-03-16 23:59:59 -const char fingerprint___github_com [] PROGMEM = "29:70:30:74:ca:3c:48:f5:4a:79:c6:2d:11:57:a2:41:2a:2d:7d:5c"; +// not valid before: 2023-02-16 00:00:00 +// not valid after: 2024-03-15 23:59:59 +const char fingerprint___github_com [] PROGMEM = "2d:79:6c:90:2d:ad:8a:2e:4f:d1:e2:99:ed:e8:91:29:36:40:f8:58"; const char pubkey___github_com [] PROGMEM = R"PUBKEY( -----BEGIN PUBLIC KEY----- -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAESG9YDaGa2BwrfubzcAQFOjMd71wz -8ofipEwn5CN1GAr5/d5T4F8kG07apsv8t5emn5Fufm/uMW1iJXug/96/QQ== +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEVB9Da7ntQj93REi9YoUrTvO/wrkz +xLleXrDjJGe1OI3tsrk+cWbZYAbwL6unbhZsAqbyBcr5BHK5M3vxZVSGRg== -----END PUBLIC KEY----- )PUBKEY"; diff --git a/libraries/ESP8266WiFi/examples/WiFiTelnetToSerial/WiFiTelnetToSerial.ino b/libraries/ESP8266WiFi/examples/WiFiTelnetToSerial/WiFiTelnetToSerial.ino index 05bdb94aa9..b31c552e35 100644 --- a/libraries/ESP8266WiFi/examples/WiFiTelnetToSerial/WiFiTelnetToSerial.ino +++ b/libraries/ESP8266WiFi/examples/WiFiTelnetToSerial/WiFiTelnetToSerial.ino @@ -31,7 +31,7 @@ SWAP_PINS: 0: use Serial1 for logging (legacy example) 1: configure Hardware Serial port on RX:GPIO13 TX:GPIO15 - and use SoftwareSerial for logging on + and use EspSoftwareSerial for logging on standard Serial pins RX:GPIO3 and TX:GPIO1 */ @@ -84,11 +84,11 @@ void setup() { #if SWAP_PINS Serial.swap(); // Hardware serial is now on RX:GPIO13 TX:GPIO15 - // use SoftwareSerial on regular RX(3)/TX(1) for logging + // use EspSoftwareSerial on regular RX(3)/TX(1) for logging logger = new SoftwareSerial(3, 1); logger->begin(BAUD_LOGGER); logger->enableIntTx(false); - logger->println("\n\nUsing SoftwareSerial for logging"); + logger->println("\n\nUsing EspSoftwareSerial for logging"); #else logger->begin(BAUD_LOGGER); logger->println("\n\nUsing Serial1 for logging"); diff --git a/libraries/ESP8266WiFi/src/ESP8266WiFiGeneric.cpp b/libraries/ESP8266WiFi/src/ESP8266WiFiGeneric.cpp index be482737a8..f64e6d3ac5 100644 --- a/libraries/ESP8266WiFi/src/ESP8266WiFiGeneric.cpp +++ b/libraries/ESP8266WiFi/src/ESP8266WiFiGeneric.cpp @@ -451,9 +451,8 @@ bool ESP8266WiFiGenericClass::mode(WiFiMode_t m) { //tasks to wait correctly. constexpr unsigned int timeoutValue = 1000; //1 second if(can_yield()) { - // The final argument, intvl_ms, to esp_delay influences how frequently - // the scheduled recurrent functions (Schedule.h) are probed. - esp_delay(timeoutValue, [m]() { return wifi_get_opmode() != m; }, 5); + // check opmode every 100ms or give up after timeout + esp_delay(timeoutValue, [m]() { return wifi_get_opmode() != m; }, 100); //if at this point mode still hasn't been reached, give up if(wifi_get_opmode() != (uint8) m) { @@ -642,11 +641,8 @@ static int hostByNameImpl(const char* aHostname, IPAddress& aResult, uint32_t ti // We need to wait for c/b to fire *or* we exit on our own timeout // (which also requires us to notify the c/b that it is supposed to delete the pending obj) case ERR_INPROGRESS: - // Re-check every 10ms, we expect this to happen fast - esp_delay(timeout_ms, - [&]() { - return !pending->done; - }, 10); + // sleep until dns_found_callback is called or timeout is reached + esp_delay(timeout_ms, [&]() { return !pending->done; }); if (pending->done) { if ((pending->addr).isSet()) { diff --git a/libraries/ESP8266WiFi/src/ESP8266WiFiMulti.cpp b/libraries/ESP8266WiFi/src/ESP8266WiFiMulti.cpp index dada704ab1..bcd433e1b1 100644 --- a/libraries/ESP8266WiFi/src/ESP8266WiFiMulti.cpp +++ b/libraries/ESP8266WiFi/src/ESP8266WiFiMulti.cpp @@ -84,13 +84,13 @@ static void printWiFiStatus(wl_status_t status) static wl_status_t waitWiFiConnect(uint32_t connectTimeoutMs) { wl_status_t status = WL_CONNECT_FAILED; - // The final argument, intvl_ms, to esp_delay influences how frequently - // the scheduled recurrent functions (Schedule.h) are probed. + // Wait for WiFi to connect + // stop waiting upon status checked every 100ms or when timeout is reached esp_delay(connectTimeoutMs, [&status]() { status = WiFi.status(); return status != WL_CONNECTED && status != WL_CONNECT_FAILED; - }, 0); + }, 100); // Check status if (status == WL_CONNECTED) { @@ -236,13 +236,12 @@ int8_t ESP8266WiFiMulti::startScan() WiFi.scanNetworks(true); // Wait for WiFi scan change or timeout - // The final argument, intvl_ms, to esp_delay influences how frequently - // the scheduled recurrent functions (Schedule.h) are probed. + // stop waiting upon status checked every 100ms or when timeout is reached esp_delay(WIFI_SCAN_TIMEOUT_MS, [&scanResult]() { scanResult = WiFi.scanComplete(); return scanResult < 0; - }, 0); + }, 100); // Check for scan timeout which may occur when scan does not report completion if (scanResult < 0) { DEBUG_WIFI_MULTI("[WIFIM] Scan timeout\n"); diff --git a/libraries/ESP8266WiFi/src/include/ClientContext.h b/libraries/ESP8266WiFi/src/include/ClientContext.h index 7fcad3678a..5c579c8848 100644 --- a/libraries/ESP8266WiFi/src/include/ClientContext.h +++ b/libraries/ESP8266WiFi/src/include/ClientContext.h @@ -144,8 +144,7 @@ class ClientContext _connect_pending = true; _op_start_time = millis(); // will resume on timeout or when _connected or _notify_error fires - // give scheduled functions a chance to run (e.g. Ethernet uses recurrent) - esp_delay(_timeout_ms, [this]() { return this->_connect_pending; }, 1); + esp_delay(_timeout_ms, [this]() { return this->_connect_pending; }); _connect_pending = false; if (!_pcb) { DEBUGV(":cabrt\r\n"); @@ -485,8 +484,7 @@ class ClientContext _send_waiting = true; // will resume on timeout or when _write_some_from_cb or _notify_error fires - // give scheduled functions a chance to run (e.g. Ethernet uses recurrent) - esp_delay(_timeout_ms, [this]() { return this->_send_waiting; }, 1); + esp_delay(_timeout_ms, [this]() { return this->_send_waiting; }); _send_waiting = false; } while(true); diff --git a/libraries/SD/src/SD.cpp b/libraries/SD/src/SD.cpp index a3f0431dd8..45302fb8a7 100644 --- a/libraries/SD/src/SD.cpp +++ b/libraries/SD/src/SD.cpp @@ -1,5 +1,14 @@ #include "SD.h" +static_assert(__builtin_strcmp(SDClassFileMode(FILE_READ), "r") == 0, ""); +static_assert(__builtin_strcmp(SDClassFileMode(FILE_WRITE), "a+") == 0, ""); + +static_assert(__builtin_strcmp(SDClassFileMode(O_RDONLY), "r") == 0, ""); +static_assert(__builtin_strcmp(SDClassFileMode(O_WRONLY), "w+") == 0, ""); +static_assert(__builtin_strcmp(SDClassFileMode(O_RDWR), "w+") == 0, ""); +static_assert(__builtin_strcmp(SDClassFileMode(O_WRONLY | O_APPEND), "a") == 0, ""); +static_assert(__builtin_strcmp(SDClassFileMode(O_RDWR | O_APPEND), "a+") == 0, ""); + #if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_SD) SDClass SD; #endif diff --git a/libraries/SD/src/SD.h b/libraries/SD/src/SD.h index 73e9e76f2b..8511dfe584 100644 --- a/libraries/SD/src/SD.h +++ b/libraries/SD/src/SD.h @@ -24,11 +24,44 @@ #include #include +// Avoid type ambiguity, force u8 instead of untyped literal +// ref. #6106 as to why we add APPEND to WRITE + +inline constexpr uint8_t SDClassFileRead { FILE_READ }; #undef FILE_READ -#define FILE_READ ((uint8_t)O_READ) -#undef FILE_WRITE -#define FILE_WRITE ((uint8_t)(O_READ | O_WRITE | O_CREAT | O_APPEND)) +#define FILE_READ SDClassFileRead +inline constexpr uint8_t SDClassFileWrite { FILE_WRITE | O_APPEND }; +#undef FILE_WRITE +#define FILE_WRITE SDClassFileWrite + +static inline constexpr const char* SDClassFileMode(uint8_t mode) { + bool read = false; + bool write = false; + + switch (mode & O_ACCMODE) { + case O_RDONLY: + read = true; + break; + case O_WRONLY: + write = true; + break; + case O_RDWR: + read = true; + write = true; + break; + } + + const bool append = (mode & O_APPEND) > 0; + + if ( read && !write ) { return "r"; } + else if ( !read && write && !append ) { return "w+"; } + else if ( !read && write && append ) { return "a"; } + else if ( read && write && !append ) { return "w+"; } // may be a bug in FS::mode interpretation, "r+" seems proper + else if ( read && write && append ) { return "a+"; } + + return "r"; +} class SDClass { public: @@ -45,7 +78,7 @@ class SDClass { } fs::File open(const char *filename, uint8_t mode = FILE_READ) { - return SDFS.open(filename, getMode(mode)); + return SDFS.open(filename, SDClassFileMode(mode)); } fs::File open(const char *filename, const char *mode) { @@ -158,18 +191,6 @@ class SDClass { } private: - const char *getMode(uint8_t mode) { - bool read = (mode & O_READ) ? true : false; - bool write = (mode & O_WRITE) ? true : false; - bool append = (mode & O_APPEND) ? true : false; - if ( read & !write ) { return "r"; } - else if ( !read & write & !append ) { return "w+"; } - else if ( !read & write & append ) { return "a"; } - else if ( read & write & !append ) { return "w+"; } // may be a bug in FS::mode interpretation, "r+" seems proper - else if ( read & write & append ) { return "a+"; } - else { return "r"; } - } - static time_t wrapperTimeCB(void) { extern void (*__SD__userDateTimeCB)(uint16_t*, uint16_t*); if (__SD__userDateTimeCB) { diff --git a/libraries/SoftwareSerial b/libraries/SoftwareSerial index 56d19bf5fe..a00554a6ad 160000 --- a/libraries/SoftwareSerial +++ b/libraries/SoftwareSerial @@ -1 +1 @@ -Subproject commit 56d19bf5fe9b38cdd438e06e1396b4a341d86274 +Subproject commit a00554a6ad1d28c633dd893e6d6ec4ca2811437f diff --git a/libraries/Ticker/src/Ticker.cpp b/libraries/Ticker/src/Ticker.cpp index baac355ca5..8c45ffed8b 100644 --- a/libraries/Ticker/src/Ticker.cpp +++ b/libraries/Ticker/src/Ticker.cpp @@ -115,6 +115,11 @@ void Ticker::_static_callback() } } + // it is technically allowed to call either schedule or detach + // *during* callback execution. allow both to happen + decltype(_callback) tmp; + std::swap(tmp, _callback); + std::visit([](auto&& callback) { using T = std::decay_t; if constexpr (std::is_same_v) { @@ -122,7 +127,16 @@ void Ticker::_static_callback() } else if constexpr (std::is_same_v) { callback(); } - }, _callback); + }, tmp); + + // ...and move ourselves back only when object is in a valid state + // * ticker was not detached, zeroing timer pointer + // * nothing else replaced callback variant + if ((_timer == nullptr) || !std::holds_alternative(_callback)) { + return; + } + + std::swap(tmp, _callback); if (_repeat) { if (_tick) { diff --git a/libraries/esp8266/examples/LowPowerDemo/LowPowerDemo.ino b/libraries/esp8266/examples/LowPowerDemo/LowPowerDemo.ino index 9a6fbc5bfd..5d1fb2fb44 100644 --- a/libraries/esp8266/examples/LowPowerDemo/LowPowerDemo.ino +++ b/libraries/esp8266/examples/LowPowerDemo/LowPowerDemo.ino @@ -128,7 +128,7 @@ void setup() { // Read previous resets (Deep Sleeps) from RTC memory, if any uint32_t crcOfData = crc32((uint8_t*)&nv->rtcData.rstCount, sizeof(nv->rtcData.rstCount)); - if ((crcOfData = nv->rtcData.crc32) && (resetCause == "Deep-Sleep Wake")) { + if ((crcOfData == nv->rtcData.crc32) && (resetCause == "Deep-Sleep Wake")) { resetCount = nv->rtcData.rstCount; // read the previous reset count resetCount++; } diff --git a/libraries/esp8266/examples/SerialStress/SerialStress.ino b/libraries/esp8266/examples/SerialStress/SerialStress.ino index 4a8b6e4f45..5d6aaf2366 100644 --- a/libraries/esp8266/examples/SerialStress/SerialStress.ino +++ b/libraries/esp8266/examples/SerialStress/SerialStress.ino @@ -2,7 +2,7 @@ /* Serial read/write/verify/benchmark Using internal loopback - Using SoftwareSerial library for logging + Using EspSoftwareSerial library for logging Sketch meant for debugging only Released to public domain diff --git a/libraries/lwIP_Ethernet/library.properties b/libraries/lwIP_Ethernet/library.properties index 38b161c26e..0c38f07ef0 100644 --- a/libraries/lwIP_Ethernet/library.properties +++ b/libraries/lwIP_Ethernet/library.properties @@ -1,4 +1,4 @@ -name=lwIP-Ethernet +name=lwIP_Ethernet version=1 author=esp8266/Arduino maintainer=esp8266/Arduino diff --git a/libraries/lwIP_w5100/library.properties b/libraries/lwIP_w5100/library.properties index f0296a9265..b74c5ba56e 100644 --- a/libraries/lwIP_w5100/library.properties +++ b/libraries/lwIP_w5100/library.properties @@ -1,4 +1,4 @@ -name=lwIP_w5500 +name=lwIP_w5100 version=1 author=Nicholas Humfrey maintainer=esp8266/Arduino diff --git a/package.json b/package.json index 891d4509b8..f6b48f83ca 100644 --- a/package.json +++ b/package.json @@ -2,5 +2,5 @@ "name": "framework-arduinoespressif8266", "description": "Arduino Wiring-based Framework (ESP8266 Core)", "url": "https://github.com/esp8266/Arduino", - "version": "3.1.0" + "version": "3.1.2" } diff --git a/package/README.md b/package/README.md index dfad8f5d9d..de6f8382ab 100644 --- a/package/README.md +++ b/package/README.md @@ -73,7 +73,7 @@ Here is a rough overview of the effective release process. See the section below * Since most changes are integrated into master using squash-rebase policy (i.e. one commit per PR), `git log --oneline` gives a good overview of changes in the release. - * Prepare release notes in Markdown format. + * Prepare release notes in Markdown format. Either use the `git log --oneline` output and sort through it manually, or use Github draft release and press 'Generate release notes' (see https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes) * For changes that are breaking, duplicate those changes and put the duplicate lines into a separate group called Breaking Changes. That group should go at the top of the Changelog. The original lines for the breaking changes should be marked by appending "(Breaking change)" to the line. Example: @@ -99,11 +99,11 @@ Here is a rough overview of the effective release process. See the section below - Documentation - Boards - * Not all commit descriptions which come from `git log` will explain changes well. Reword items as necessary, with the goal that a general user of this project should be able to understand what the change is related to. Preserve references to PRs or issues (`#XXXX`). + * Not all commit descriptions which come from `git log` or PR titles will explain changes well. Reword items as necessary, with the goal that a general user of this project should be able to understand what the change is related to. Preserve references to PRs or issues (`#XXXX`). * Aggregate minor fixes (e.g. typos, small documentation changes) in a few items. Focus on preparing a good overview of the release for the users, rather than mentioning every change. - * When done, put release notes into a private [Gist](https://gist.github.com) or [firepad](https://demo.firepad.io) and send the link to other maintainers for review. + * When done, put release notes into a private [Gist](https://gist.github.com) or [HedgeDoc note](https://hedgedoc.org/) and send the link to other maintainers for review. The following points assume work in a direct clone of the repository, and not in a personal fork. @@ -141,13 +141,13 @@ The following points assume work in a direct clone of the repository, and not in 8. Check that the new (draft) release has been created (no editing at this point!), see https://github.com/esp8266/Arduino/releases. -9. Check that the boards manager package .zip file has been successfully uploaded as a release artifact. +9. Check that the boards manager package .zip file has been successfully uploaded as a release asset. 10. Check that the package index downloaded from https://arduino.esp8266.com/stable/package_esp8266com_index.json contains an entry for the new version (it may not be the first one). 11. Navigate to release list in Github here https://github.com/esp8266/Arduino/releases, press "Edit" button to edit release description, paste release notes, and publish it. -12. In the issue tracker, remove "staged-for-release" label for all issues which have it, and close them. Close the milestone associated with the released version (the milestone should be empty per point 2 above) +12. Close the milestone associated with the released version (the milestone should be empty per point 1 above) 13. Check that https://arduino-esp8266.readthedocs.io/en/latest/ has a new doc build for the new tag, and that "stable" points to that build. If a new build did not trigger, log into readthedoc's home here https://readthedocs.org/ (account must have been added to project as maintainer) and trigger it manually. @@ -186,13 +186,13 @@ The following points assume work in a direct clone of the repository, and not in - [ ] 8. Check that the new (draft) release has been created (no editing at this point!), see https://github.com/esp8266/Arduino/releases. -- [ ] 9. Check that the boards manager package .zip file has been successfully uploaded as a release artifact. +- [ ] 9. Check that the boards manager package .zip file has been successfully uploaded as a release asset. - [ ] 10. Check that the package index downloaded from https://arduino.esp8266.com/stable/package_esp8266com_index.json contains an entry for the new version (it may not be the first one). - [ ] 11. Navigate to [release list in Github](https://github.com/esp8266/Arduino/releases), press "Edit" button to edit release description, paste release notes, and publish it. -- [ ] 12. In the issue tracker, remove "staged-for-release" label for all issues which have it, and close them. Close the milestone associated with the released version (the milestone should be empty per point 1 above) +- [ ] 12. Close the milestone associated with the released version (the milestone should be empty per point 1 above) - [ ] 13. Check that https://arduino-esp8266.readthedocs.io/en/latest/ has a new doc build for the new tag, and that "stable" points to that build. If a new build did not trigger, log into readthedoc's home here https://readthedocs.org/ (account must have been added to project as maintainer) and trigger it manually. @@ -202,3 +202,35 @@ The following points assume work in a direct clone of the repository, and not in * In main README.md go to "Latest release" section, change version number in the readthedocs link to the version which was just released, and verify that all links work. --------------COPY ABOVE THIS LINE-------------- ``` + +## Updating a SSH deploy key + +A SSH private/public key pair is required to update the master JSON (the final step of the release process). Sometimes GitHub will expire one side or the other of that key, and a new one will need to be regenerated and installed in the https://github.com/esp8266/esp8266.github.io (JSON) and https://github.com/esp8266/Arduino (core) repos. + +1. Generate a new public/private SSH key pair with an empty passphrase: +```console +$ ssh-keygen -f deploy_key -t ed25519 -N '' -C earlephilhower@yahoo.com (**replace with your GH user account email**) +Generating public/private ed25519 key pair. +Your identification has been saved in deploy_key +Your public key has been saved in deploy_key.pub +The key fingerprint is: +... +``` + +2. Copy the contents of `deploy_key.pub` to the clipboard: +```console +$ cat deploy_key.pub +ssh-ed25519 AAA..... earlephilhower@yahoo.com +``` + +3. Install the deploy key for esp8266.github.io repository. Go to https://github.com/esp8266/esp8266.github.io and the `Settings->Deploy Keys` and `Add deploy key`. Paste the (public key) string into the box and select `Allow writes` and hit OK. + +4. Convert the `deploy_key` private key to a 1-line base64 representation and copy it to the clipboard. +```console +$ base64 -w 0 < deploy_key && echo "" +yEvYm..... (**note this must be one single long line, hence the "-w 0"**) +``` + +5. Install the private key to the Core repo. Go to https://github.com/esp8266/Arduino and select `Settings->Secrets->Actions` and add or update a `Repository secret` called `GHCI_DEPLOY_KEY`. Paste the 1-line base64 contents of your clipboard to the box and hit OK. + +6. If the release failed in the `Update master JSON file` action, from the GitHub web interface run the `Actions->Release XXX->Re-run failed jobs` to re-run it and check its output. diff --git a/platform.txt b/platform.txt index 5334d3ee2e..3b55c024a7 100644 --- a/platform.txt +++ b/platform.txt @@ -5,8 +5,8 @@ # For more info: # https://github.com/arduino/Arduino/wiki/Arduino-IDE-1.5-3rd-party-Hardware-specification -name=ESP8266 Boards (3.1.0) -version=3.1.0 +name=ESP8266 Boards (3.1.2) +version=3.1.2 # These will be removed by the packager script when doing a JSON release runtime.tools.xtensa-lx106-elf-gcc.path={runtime.platform.path}/tools/xtensa-lx106-elf @@ -76,18 +76,18 @@ compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_S compiler.libraries.ldflags= compiler.c.cmd=xtensa-lx106-elf-gcc -compiler.c.flags=-c "{compiler.warning_flags}-gcc" -std=gnu17 {build.stacksmash_flags} -Os -g -free -fipa-pta -Werror=return-type -Wpointer-arith -Wno-implicit-function-declaration -Wl,-EL -fno-inline-functions -nostdlib -mlongcalls -mtext-section-literals -falign-functions=4 -MMD -ffunction-sections -fdata-sections {build.exception_flags} {build.sslflags} {build.mmuflags} {build.non32xferflags} +compiler.c.flags=-c "{compiler.warning_flags}-cflags" -std=gnu17 {build.stacksmash_flags} -Os -g -free -fipa-pta -Werror=return-type -Wpointer-arith -Wno-implicit-function-declaration -Wl,-EL -fno-inline-functions -nostdlib -mlongcalls -mtext-section-literals -falign-functions=4 -MMD -ffunction-sections -fdata-sections {build.exception_flags} {build.sslflags} {build.mmuflags} {build.non32xferflags} compiler.S.cmd=xtensa-lx106-elf-gcc compiler.S.flags=-c -g -x assembler-with-cpp -MMD -mlongcalls "-I{runtime.tools.xtensa-lx106-elf-gcc.path}/include/" -compiler.c.elf.flags=-g "{compiler.warning_flags}-gcc" -Os -nostdlib -Wl,--no-check-sections -u app_entry {build.float} -Wl,-static "-L{compiler.sdk.path}/lib" "-L{compiler.sdk.path}/lib/{build.sdk}" "-L{build.path}" "-L{compiler.libc.path}/lib" "-Tlocal.eagle.flash.ld" -Wl,--gc-sections -Wl,-wrap,system_restart_local -Wl,-wrap,spi_flash_read +compiler.c.elf.flags=-g "{compiler.warning_flags}-cflags" -Os -nostdlib -Wl,--no-check-sections -u app_entry {build.float} -Wl,-static "-L{compiler.sdk.path}/lib" "-L{compiler.sdk.path}/lib/{build.sdk}" "-L{build.path}" "-L{compiler.libc.path}/lib" "-Tlocal.eagle.flash.ld" -Wl,--gc-sections -Wl,-wrap,system_restart_local -Wl,-wrap,spi_flash_read compiler.c.elf.cmd=xtensa-lx106-elf-gcc compiler.c.elf.libs=-lhal -lphy -lpp -lnet80211 {build.lwip_lib} -lwpa -lcrypto -lmain -lwps -lbearssl -lespnow -lsmartconfig -lairkiss -lwpa2 {build.stdcpp_lib} -lm -lc -lgcc compiler.cpp.cmd=xtensa-lx106-elf-g++ -compiler.cpp.flags=-c "{compiler.warning_flags}-g++" {build.stacksmash_flags} -Os -g -free -fipa-pta -Werror=return-type -mlongcalls -mtext-section-literals -fno-rtti -falign-functions=4 {build.stdcpp_level} -MMD -ffunction-sections -fdata-sections {build.exception_flags} {build.sslflags} {build.mmuflags} {build.non32xferflags} +compiler.cpp.flags=-c "{compiler.warning_flags}-cppflags" {build.stacksmash_flags} -Os -g -free -fipa-pta -Werror=return-type -mlongcalls -mtext-section-literals -fno-rtti -falign-functions=4 {build.stdcpp_level} -MMD -ffunction-sections -fdata-sections {build.exception_flags} {build.sslflags} {build.mmuflags} {build.non32xferflags} compiler.as.cmd=xtensa-lx106-elf-as @@ -118,7 +118,7 @@ recipe.hooks.sketch.prebuild.pattern="{runtime.tools.python3.path}/python3" -I " recipe.hooks.prebuild.1.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.makecorever}" --build_path "{build.path}" --platform_path "{runtime.platform.path}" --version "{version}" # Handle processing sketch global options -recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} +recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} ## Build the app.ld linker file diff --git a/tests/host/common/UdpContextSocket.cpp b/tests/host/common/UdpContextSocket.cpp index c598460180..2ffa4eb22f 100644 --- a/tests/host/common/UdpContextSocket.cpp +++ b/tests/host/common/UdpContextSocket.cpp @@ -166,14 +166,14 @@ size_t mockUDPFillInBuf(int sock, char* ccinbuf, size_t& ccinbufsize, uint8_t& a return ccinbufsize += ret; } -size_t mockUDPPeekBytes(int sock, char* dst, size_t usersize, int timeout_ms, char* ccinbuf, - size_t& ccinbufsize) +size_t mockUDPPeekBytes(int sock, char* dst, size_t offset, size_t usersize, int timeout_ms, + char* ccinbuf, size_t& ccinbufsize) { (void)sock; (void)timeout_ms; - if (usersize > CCBUFSIZE) + if (offset + usersize > CCBUFSIZE) fprintf(stderr, MOCK "CCBUFSIZE(%d) should be increased by %zd bytes (-> %zd)\n", CCBUFSIZE, - usersize - CCBUFSIZE, usersize); + offset + usersize - CCBUFSIZE, offset + usersize); size_t retsize = 0; if (ccinbufsize) @@ -183,25 +183,10 @@ size_t mockUDPPeekBytes(int sock, char* dst, size_t usersize, int timeout_ms, ch if (retsize > ccinbufsize) retsize = ccinbufsize; } - memcpy(dst, ccinbuf, retsize); + memcpy(dst, ccinbuf + offset, retsize); return retsize; } -void mockUDPSwallow(size_t copied, char* ccinbuf, size_t& ccinbufsize) -{ - // poor man buffer - memmove(ccinbuf, ccinbuf + copied, ccinbufsize - copied); - ccinbufsize -= copied; -} - -size_t mockUDPRead(int sock, char* dst, size_t size, int timeout_ms, char* ccinbuf, - size_t& ccinbufsize) -{ - size_t copied = mockUDPPeekBytes(sock, dst, size, timeout_ms, ccinbuf, ccinbufsize); - mockUDPSwallow(copied, ccinbuf, ccinbufsize); - return copied; -} - size_t mockUDPWrite(int sock, const uint8_t* data, size_t size, int timeout_ms, uint32_t ipv4, uint16_t port) { diff --git a/tests/host/common/include/UdpContext.h b/tests/host/common/include/UdpContext.h index 88cb46f9ae..e8ae9461e9 100644 --- a/tests/host/common/include/UdpContext.h +++ b/tests/host/common/include/UdpContext.h @@ -117,12 +117,12 @@ class UdpContext size_t getSize() { - return _inbufsize; + return _inbufsize - _inoffset; } size_t tell() const { - return 0; + return _inoffset; } void seek(const size_t pos) @@ -132,7 +132,7 @@ class UdpContext mockverbose("UDPContext::seek too far (%zd >= %zd)\n", pos, _inbufsize); exit(EXIT_FAILURE); } - mockUDPSwallow(pos, _inbuf, _inbufsize); + _inoffset = pos; } bool isValidOffset(const size_t pos) const @@ -165,6 +165,7 @@ class UdpContext bool next() { _inbufsize = 0; + _inoffset = 0; mockUDPFillInBuf(_sock, _inbuf, _inbufsize, addrsize, addr, _dstport); if (_inbufsize > 0) { @@ -182,13 +183,16 @@ class UdpContext size_t read(char* dst, size_t size) { - return mockUDPRead(_sock, dst, size, _timeout_ms, _inbuf, _inbufsize); + //return mockUDPRead(_sock, dst, size, _timeout_ms, _inbuf, _inbufsize); + auto ret = mockUDPPeekBytes(_sock, dst, _inoffset, size, _timeout_ms, _inbuf, _inbufsize); + _inoffset += ret; + return ret; } int peek() { char c; - return mockUDPPeekBytes(_sock, &c, 1, _timeout_ms, _inbuf, _inbufsize) ?: -1; + return mockUDPPeekBytes(_sock, &c, _inoffset, 1, _timeout_ms, _inbuf, _inbufsize) ?: -1; } void flush() @@ -280,6 +284,7 @@ class UdpContext char _inbuf[CCBUFSIZE]; size_t _inbufsize = 0; + size_t _inoffset = 0; char _outbuf[CCBUFSIZE]; size_t _outbufsize = 0; diff --git a/tests/host/common/mock.h b/tests/host/common/mock.h index dbcb1dbf31..b3308282f0 100644 --- a/tests/host/common/mock.h +++ b/tests/host/common/mock.h @@ -160,13 +160,10 @@ int mockUDPSocket(); bool mockUDPListen(int sock, uint32_t dstaddr, uint16_t port, uint32_t mcast = 0); size_t mockUDPFillInBuf(int sock, char* ccinbuf, size_t& ccinbufsize, uint8_t& addrsize, uint8_t addr[16], uint16_t& port); -size_t mockUDPPeekBytes(int sock, char* dst, size_t usersize, int timeout_ms, char* ccinbuf, - size_t& ccinbufsize); -size_t mockUDPRead(int sock, char* dst, size_t size, int timeout_ms, char* ccinbuf, - size_t& ccinbufsize); +size_t mockUDPPeekBytes(int sock, char* dst, size_t offset, size_t usersize, int timeout_ms, + char* ccinbuf, size_t& ccinbufsize); size_t mockUDPWrite(int sock, const uint8_t* data, size_t size, int timeout_ms, uint32_t ipv4, uint16_t port); -void mockUDPSwallow(size_t copied, char* ccinbuf, size_t& ccinbufsize); class UdpContext; void register_udp(int sock, UdpContext* udp = nullptr); diff --git a/tests/sanity_check.sh b/tests/sanity_check.sh new file mode 100755 index 0000000000..a4754a0ed4 --- /dev/null +++ b/tests/sanity_check.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +root=$(git rev-parse --show-toplevel) +source "$root/tests/common.sh" + +pushd "$root"/tools +python3 get.py -q + +popd +pushd "$cache_dir" + +gcc="$root/tools/xtensa-lx106-elf/bin/xtensa-lx106-elf-gcc"\ +" -I$root/cores/esp8266"\ +" -I$root/tools/sdk/include"\ +" -I$root/variants/generic"\ +" -I$root/tools/sdk/libc/xtensa-lx106-elf" + +$gcc --verbose + +set -v -x + +cat << EOF > arduino.c +#include +EOF + +$gcc -c arduino.c + +cat << EOF > coredecls.c +#include +EOF + +$gcc -c coredecls.c + +cat << EOF > features.c +#include +EOF + +$gcc -c features.c + +cat << EOF > sdk.c +#include +EOF + +$gcc -c sdk.c diff --git a/tools/elf2bin.py b/tools/elf2bin.py index 035f7a93ae..c881b1d709 100755 --- a/tools/elf2bin.py +++ b/tools/elf2bin.py @@ -20,6 +20,7 @@ from __future__ import print_function import argparse +import io import re import os import subprocess @@ -33,37 +34,75 @@ crcsize_offset = 4096 + 16 crcval_offset = 4096 + 16 + 4 + +class Elf2BinException(Exception): + pass + + def get_elf_entry(elf, path): - p = subprocess.Popen([path + "/xtensa-lx106-elf-readelf", '-h', elf], stdout=subprocess.PIPE, universal_newlines=True ) - lines = p.stdout.readlines() - for line in lines: + result = subprocess.run( + [os.path.join(path, "xtensa-lx106-elf-readelf"), "-h", elf], + check=True, + timeout=15, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + + lines = io.StringIO(result.stdout) + for line in lines.readlines(): if 'Entry point address' in line: words = re.split('\s+', line) entry_point = words[-2] return int(entry_point, 16) - raise Exception('Unable to find entry point in file "' + elf + '"') + + raise Elf2BinException(f'Unable to find entry point in file "{elf}"') + def get_segment_size_addr(elf, segment, path): - p = subprocess.Popen([path + '/xtensa-lx106-elf-objdump', '-h', '-j', segment, elf], stdout=subprocess.PIPE, universal_newlines=True ) - lines = p.stdout.readlines() - for line in lines: + result = subprocess.run( + [os.path.join(path, "xtensa-lx106-elf-objdump"), "-h", "-j", segment, elf], + check=True, + timeout=15, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + + lines = io.StringIO(result.stdout) + for line in lines.readlines(): if segment in line: words = re.split('\s+', line) size = int(words[3], 16) addr = int(words[4], 16) return [ size, addr ] - raise Exception('Unable to find size and start point in file "' + elf + '" for "' + segment + '"') + + raise Elf2BinException( + f'Unable to find size and start point in file "{elf}" for "{segment}"' + ) + def read_segment(elf, segment, path): fd, tmpfile = tempfile.mkstemp() os.close(fd) - subprocess.check_call([path + "/xtensa-lx106-elf-objcopy", '-O', 'binary', '--only-section=' + segment, elf, tmpfile], stdout=subprocess.PIPE) - with open(tmpfile, "rb") as f: - raw = f.read() - os.remove(tmpfile) + try: + subprocess.check_call( + [ + os.path.join(path, "xtensa-lx106-elf-objcopy"), + "-O", + "binary", + f"--only-section={segment}", + elf, + tmpfile, + ], + stdout=subprocess.PIPE, + ) + with open(tmpfile, "rb") as f: + raw = f.read() + finally: + os.remove(tmpfile) return raw + def write_bin(out, args, elf, segments, to_addr): entry = int(get_elf_entry( elf, args.path )) header = [ 0xe9, len(segments), fmodeb[args.flash_mode], ffreqb[args.flash_freq] + 16 * fsizeb[args.flash_size], @@ -85,7 +124,7 @@ def write_bin(out, args, elf, segments, to_addr): try: for data in raw: checksum = checksum ^ ord(data) - except Exception: + except: for data in raw: checksum = checksum ^ data total_size += 1 @@ -95,11 +134,14 @@ def write_bin(out, args, elf, segments, to_addr): out.write(bytearray([checksum])) if to_addr != 0: if total_size + 8 > to_addr: - raise Exception('Bin image of ' + elf + ' is too big, actual size ' + str(total_size + 8) + ', target size ' + str(to_addr) + '.') + raise Elf2BinException( + f'Bin image of "{elf}" is too big! Actual size {str(total_size + 8)}, target size {str(to_addr)}' + ) while total_size < to_addr: out.write(bytearray([0xaa])) total_size += 1 + def crc8266(ldata): "Return the CRC of ldata using same algorithm as eboot" crc = 0xffffffff @@ -119,6 +161,7 @@ def crc8266(ldata): crc = int(crc ^ 0x04c11db7) return crc + def store_word(raw, offset, val): "Place a 4-byte word in 8266-dependent order in the raw image" raw[offset] = val & 255 @@ -127,6 +170,7 @@ def store_word(raw, offset, val): raw[offset + 3] = (val >> 24) & 255 return raw + def add_crc(out): with open(out, "rb") as binfile: raw = bytearray(binfile.read()) @@ -141,15 +185,16 @@ def add_crc(out): with open(out, "wb") as binfile: binfile.write(raw) + def gzip_bin(mode, out): import gzip firmware_path = out - gzip_path = firmware_path + '.gz' - orig_path = firmware_path + '.orig' + gzip_path = f"{firmware_path}.gz" + orig_path = f"{firmware_path}.orig" if os.path.exists(gzip_path): os.remove(gzip_path) - print('GZipping firmware ' + firmware_path) + print(f'GZipping firmware {firmware_path}') with open(firmware_path, 'rb') as firmware_file, \ gzip.open(gzip_path, 'wb') as dest: data = firmware_file.read() @@ -162,10 +207,11 @@ def gzip_bin(mode, out): if mode == "PIO": if os.path.exists(orig_path): os.remove(orig_path) - print('Moving original firmware to ' + orig_path) + print(f'Moving original firmware to {orig_path}') os.rename(firmware_path, orig_path) os.rename(gzip_path, firmware_path) + def main(): parser = argparse.ArgumentParser(description='Create a BIN file from eboot.elf and Arduino sketch.elf for upload by esptool.py') parser.add_argument('-e', '--eboot', action='https://github.com/store', required=True, help='Path to the Arduino eboot.elf bootloader') @@ -179,8 +225,7 @@ def main(): args = parser.parse_args() - print('Creating BIN file "{out}" using "{eboot}" and "{app}"'.format( - out=args.out, eboot=args.eboot, app=args.app)) + print(f'Creating BIN file "{args.out}" using "{args.eboot}" and "{args.app}"') with open(args.out, "wb") as out: def wrapper(**kwargs): diff --git a/tools/mkbuildoptglobals.py b/tools/mkbuildoptglobals.py index 23a9e940e1..62a3373aee 100644 --- a/tools/mkbuildoptglobals.py +++ b/tools/mkbuildoptglobals.py @@ -81,7 +81,7 @@ build.opt.fqfn={build.path}/core/build.opt mkbuildoptglobals.extra_flags= -recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} +recipe.hooks.prebuild.2.pattern="{runtime.tools.python3.path}/python3" -I "{runtime.tools.mkbuildoptglobals}" "{runtime.ide.path}" {runtime.ide.version} "{build.path}" "{build.opt.fqfn}" "{globals.h.source.fqfn}" "{commonhfile.fqfn}" {mkbuildoptglobals.extra_flags} compiler.cpreprocessor.flags=-D__ets__ -DICACHE_FLASH -U__STRICT_ANSI__ -D_GNU_SOURCE -DESP8266 @{build.opt.path} "-I{compiler.sdk.path}/include" "-I{compiler.sdk.path}/{build.lwip_include}" "-I{compiler.libc.path}/include" "-I{build.path}/core" """ @@ -183,13 +183,24 @@ """ import argparse -from shutil import copyfile import glob +import locale import os import platform import sys import textwrap import time +import traceback + +from shutil import copyfile + + +# Stay in sync with our bundled version +PYTHON_REQUIRES = (3, 7) + +if sys.version_info < PYTHON_REQUIRES: + raise SystemExit(f"{__file__}\nMinimal supported version of Python is {PYTHON_REQUIRES[0]}.{PYTHON_REQUIRES[1]}") + # Need to work on signature line used for match to avoid conflicts with # existing embedded documentation methods. @@ -201,6 +212,7 @@ err_print_flag = False msg_print_buf = "" debug_enabled = False +default_encoding = None # Issues trying to address through buffered printing # 1. Arduino IDE 2.0 RC5 does not show stderr text in color. Text printed does @@ -295,16 +307,16 @@ def copy_create_build_file(source_fqfn, build_target_fqfn): pass return True # file changed - def add_include_line(build_opt_fqfn, include_fqfn): + global default_encoding if not os.path.exists(include_fqfn): # If file is missing, we need an place holder - with open(include_fqfn, 'w', encoding="utf-8"): + with open(include_fqfn, 'w', encoding=default_encoding): pass - print("add_include_line: Created " + include_fqfn) - with open(build_opt_fqfn, 'a', encoding="utf-8") as build_opt: - build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n') + print_msg("add_include_line: Created " + include_fqfn) + with open(build_opt_fqfn, 'a', encoding=default_encoding) as build_opt: + build_opt.write('-include "' + include_fqfn.replace('\\', '\\\\') + '"\n') def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn): """ @@ -313,8 +325,9 @@ def extract_create_build_opt_file(globals_h_fqfn, file_name, build_opt_fqfn): copy of Sketch.ino.globals.h. """ global build_opt_signature + global default_encoding - build_opt = open(build_opt_fqfn, 'w', encoding="utf-8") + build_opt = open(build_opt_fqfn, 'w', encoding=default_encoding) if not os.path.exists(globals_h_fqfn) or (0 == os.path.getsize(globals_h_fqfn)): build_opt.close() return False @@ -416,68 +429,6 @@ def discover_1st_time_run(build_path): return 0 == count -def find_preferences_txt(runtime_ide_path): - """ - Check for perferences.txt in well-known locations. Most OSs have two - possibilities. When "portable" is present, it takes priority. Otherwise, the - remaining path wins. However, Windows has two. Depending on the install - source, the APP store or website download, both may appear and create an - ambiguous result. - - Return two item list - Two non "None" items indicate an ambiguous state. - - OS Path list for Arduino IDE 1.6.0 and newer - from: https://www.arduino.cc/en/hacking/preferences - """ - platform_name = platform.system() - if "Linux" == platform_name: - # Test for portable 1ST - # /portable/preferences.txt (when used in portable mode) - # For more on portable mode see https://docs.arduino.cc/software/ide-v1/tutorials/PortableIDE - fqfn = os.path.normpath(runtime_ide_path + "/portable/preferences.txt") - # Linux - verified with Arduino IDE 1.8.19 - if os.path.exists(fqfn): - return [fqfn, None] - fqfn = os.path.expanduser("~/.arduino15/preferences.txt") - # Linux - verified with Arduino IDE 1.8.18 and 2.0 RC5 64bit and AppImage - if os.path.exists(fqfn): - return [fqfn, None] - elif "Windows" == platform_name: - fqfn = os.path.normpath(runtime_ide_path + "\portable\preferences.txt") - # verified on Windows 10 with Arduino IDE 1.8.19 - if os.path.exists(fqfn): - return [fqfn, None] - # It is never simple. Arduino from the Windows APP store or the download - # Windows 8 and up option will save "preferences.txt" in one location. - # The downloaded Windows 7 (and up version) will put "preferences.txt" - # in a different location. When both are present due to various possible - # scenarios, use the more modern. - fqfn = os.path.expanduser("~\Documents\ArduinoData\preferences.txt") - # Path for "Windows app" - verified on Windows 10 with Arduino IDE 1.8.19 from APP store - fqfn2 = os.path.expanduser("~\AppData\local\Arduino15\preferences.txt") - # Path for Windows 7 and up - verified on Windows 10 with Arduino IDE 1.8.19 - if os.path.exists(fqfn): - if os.path.exists(fqfn2): - print_err("Multiple 'preferences.txt' files found:") - print_err(" " + fqfn) - print_err(" " + fqfn2) - return [fqfn, None] - else: - return [fqfn, fqfn2] - elif os.path.exists(fqfn2): - return [fqfn2, None] - elif "Darwin" == platform_name: - # Portable is not compatable with Mac OS X - # see https://docs.arduino.cc/software/ide-v1/tutorials/PortableIDE - fqfn = os.path.expanduser("~/Library/Arduino15/preferences.txt") - # Mac OS X - unverified - if os.path.exists(fqfn): - return [fqfn, None] - - print_err("File preferences.txt not found on " + platform_name) - return [None, None] - - def get_preferences_txt(file_fqfn, key): # Get Key Value, key is allowed to be missing. # We assume file file_fqfn exists @@ -505,40 +456,28 @@ def check_preferences_txt(runtime_ide_path, preferences_file): else: print_err(f"Override preferences file '{preferences_file}' not found.") - elif runtime_ide_path != None: - # For a particular install, search the expected locations for platform.txt - # This should never fail. - file_fqfn = find_preferences_txt(runtime_ide_path) - if file_fqfn[0] != None: - print_msg(f"Using preferences from '{file_fqfn[0]}'") - val0 = get_preferences_txt(file_fqfn[0], key) - val1 = val0 - if file_fqfn[1] != None: - val1 = get_preferences_txt(file_fqfn[1], key) - if val0 == val1: # We can safely ignore that there were two preferences.txt files - return val0 - else: - print_err(f"Found too many preferences.txt files with different values for '{key}'") - raise UserWarning - else: - # Something is wrong with the installation or our understanding of the installation. - print_err("'preferences.txt' file missing from well known locations.") - - return None - + # Referencing the preferences.txt for an indication of shared "core.a" + # caching is unreliable. There are too many places reference.txt can be + # stored and no hints of which the Arduino build might be using. Unless + # directed otherwise, assume "core.a" caching true. + print_msg(f"Assume aggressive 'core.a' caching enabled.") + return True def touch(fname, times=None): with open(fname, "ab") as file: - os.utime(file.fileno(), times) + file.close(); + os.utime(fname, times) def synchronous_touch(globals_h_fqfn, commonhfile_fqfn): global debug_enabled # touch both files with the same timestamp touch(globals_h_fqfn) with open(globals_h_fqfn, "rb") as file: - ts = os.stat(file.fileno()) - with open(commonhfile_fqfn, "ab") as file2: - os.utime(file2.fileno(), ns=(ts.st_atime_ns, ts.st_mtime_ns)) + file.close() + with open(commonhfile_fqfn, "ab") as file2: + file2.close() + ts = os.stat(globals_h_fqfn) + os.utime(commonhfile_fqfn, ns=(ts.st_atime_ns, ts.st_mtime_ns)) if debug_enabled: print_dbg("After synchronous_touch") @@ -552,12 +491,8 @@ def synchronous_touch(globals_h_fqfn, commonhfile_fqfn): def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): global docs_url print_dbg(f"runtime_ide_version: {args.runtime_ide_version}") - if args.runtime_ide_version < 10802: # CI also has version 10607 -- and args.runtime_ide_version != 10607: - # Aggresive core caching - not implemented before version 1.8.2 - # Note, Arduino IDE 2.0 rc5 has version 1.6.7 and has aggressive caching. - print_dbg(f"Old version ({args.runtime_ide_version}) of Arduino IDE no aggressive caching option") - return False - elif args.cache_core != None: + + if args.cache_core != None: print_msg(f"Preferences override, this prebuild script assumes the 'compiler.cache_core' parameter is set to {args.cache_core}") print_msg(f"To change, modify 'mkbuildoptglobals.extra_flags=(--cache_core | --no_cache_core)' in 'platform.local.txt'") return args.cache_core @@ -591,19 +526,7 @@ def determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn): preferences_fqfn = os.path.expanduser(preferences_fqfn) print_dbg(f"determine_cache_state: preferences_fqfn: {preferences_fqfn}") - try: - caching_enabled = check_preferences_txt(ide_path, preferences_fqfn) - except UserWarning: - if os.path.exists(source_globals_h_fqfn): - caching_enabled = None - print_err(f" runtime_ide_version: {args.runtime_ide_version}") - print_err(f" This must be resolved to use '{globals_name}'") - print_err(f" Read more at {docs_url}") - else: - # We can quietly ignore the problem because we are not needed. - caching_enabled = True - - return caching_enabled + return check_preferences_txt(ide_path, preferences_fqfn) """ @@ -695,12 +618,63 @@ def parse_args(): # ref nargs='*'', https://stackoverflow.com/a/4480202 # ref no '--n' parameter, https://stackoverflow.com/a/21998252 + +# retrieve *system* encoding, not the one used by python internally +if sys.version_info >= (3, 11): + def get_encoding(): + return locale.getencoding() +else: + def get_encoding(): + return locale.getdefaultlocale()[1] + + +def show_value(desc, value): + print_dbg(f'{desc:<40} {value}') + return + +def locale_dbg(): + show_value("get_encoding()", get_encoding()) + show_value("locale.getdefaultlocale()", locale.getdefaultlocale()) + show_value('sys.getfilesystemencoding()', sys.getfilesystemencoding()) + show_value("sys.getdefaultencoding()", sys.getdefaultencoding()) + show_value("locale.getpreferredencoding(False)", locale.getpreferredencoding(False)) + try: + show_value("locale.getpreferredencoding()", locale.getpreferredencoding()) + except: + pass + show_value("sys.stdout.encoding", sys.stdout.encoding) + + # use current setting + show_value("locale.setlocale(locale.LC_ALL, None)", locale.setlocale(locale.LC_ALL, None)) + try: + show_value("locale.getencoding()", locale.getencoding()) + except: + pass + show_value("locale.getlocale()", locale.getlocale()) + + # use user setting + show_value("locale.setlocale(locale.LC_ALL, '')", locale.setlocale(locale.LC_ALL, '')) + # show_value("locale.getencoding()", locale.getencoding()) + show_value("locale.getlocale()", locale.getlocale()) + return + + def main(): global build_opt_signature global docs_url global debug_enabled + global default_encoding num_include_lines = 1 + # Given that GCC will handle lines from an @file as if they were on + # the command line. I assume that the contents of @file need to be encoded + # to match that of the shell running GCC runs. I am not 100% sure this API + # gives me that, but it appears to work. + # + # However, elsewhere when dealing with source code we continue to use 'utf-8', + # ref. https://gcc.gnu.org/onlinedocs/cpp/Character-sets.html + default_encoding = get_encoding() + args = parse_args() debug_enabled = args.debug runtime_ide_path = os.path.normpath(args.runtime_ide_path) @@ -713,6 +687,11 @@ def main(): build_path_core, build_opt_name = os.path.split(build_opt_fqfn) globals_h_fqfn = os.path.join(build_path_core, globals_name) + if debug_enabled: + locale_dbg() + + print_msg(f'default_encoding: {default_encoding}') + print_dbg(f"runtime_ide_path: {runtime_ide_path}") print_dbg(f"runtime_ide_version: {args.runtime_ide_version}") print_dbg(f"build_path: {build_path}") @@ -741,13 +720,14 @@ def main(): print_dbg("First run since Arduino IDE started.") use_aggressive_caching_workaround = determine_cache_state(args, runtime_ide_path, source_globals_h_fqfn) - if use_aggressive_caching_workaround == None: - # Specific rrror messages already buffered - handle_error(1) print_dbg(f"first_time: {first_time}") print_dbg(f"use_aggressive_caching_workaround: {use_aggressive_caching_workaround}") + if not os.path.exists(build_path_core): + os.makedirs(build_path_core) + print_msg("Clean build, created dir " + build_path_core) + if first_time or \ not use_aggressive_caching_workaround or \ not os.path.exists(commonhfile_fqfn): @@ -760,10 +740,6 @@ def main(): touch(commonhfile_fqfn) print_err(f"Neutralized future timestamp on build file: {commonhfile_fqfn}") - if not os.path.exists(build_path_core): - os.makedirs(build_path_core) - print_msg("Clean build, created dir " + build_path_core) - if os.path.exists(source_globals_h_fqfn): print_msg("Using global include from " + source_globals_h_fqfn) @@ -843,4 +819,10 @@ def main(): handle_error(0) # commit print buffer if __name__ == '__main__': - sys.exit(main()) + rc = 1 + try: + rc = main() + except: + print_err(traceback.format_exc()) + handle_error(0) + sys.exit(rc) diff --git a/tools/sizes.py b/tools/sizes.py index 3bbc537faf..34ebe30755 100755 --- a/tools/sizes.py +++ b/tools/sizes.py @@ -21,9 +21,20 @@ import os import subprocess import sys +import locale sys.stdout = sys.stderr + +# retrieve *system* encoding, not the one used by python internally +if sys.version_info >= (3, 11): + def get_encoding(): + return locale.getencoding() +else: + def get_encoding(): + return locale.getdefaultlocale()[1] + + def get_segment_sizes(elf, path, mmu): iram_size = 0 iheap_size = 0 @@ -73,7 +84,7 @@ def get_segment_sizes(elf, path, mmu): ) cmd = [os.path.join(path, "xtensa-lx106-elf-size"), "-A", elf] - with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True) as proc: + with subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True, encoding=get_encoding()) as proc: lines = proc.stdout.readlines() for line in lines: words = line.split() diff --git a/tools/upload.py b/tools/upload.py index aee6add8f1..760d4dbbef 100755 --- a/tools/upload.py +++ b/tools/upload.py @@ -65,7 +65,7 @@ try: esptool.main(cmdline) -except esptool.FatalError as e: +except Exception as e: sys.stderr.write('\nA fatal esptool.py error occurred: %s' % e) finally: if erase_file: diff --git a/tools/warnings/default-g++ b/tools/warnings/default-cflags similarity index 100% rename from tools/warnings/default-g++ rename to tools/warnings/default-cflags diff --git a/tools/warnings/default-gcc b/tools/warnings/default-cppflags similarity index 100% rename from tools/warnings/default-gcc rename to tools/warnings/default-cppflags diff --git a/tools/warnings/extra-g++ b/tools/warnings/extra-cflags similarity index 100% rename from tools/warnings/extra-g++ rename to tools/warnings/extra-cflags diff --git a/tools/warnings/extra-gcc b/tools/warnings/extra-cppflags similarity index 100% rename from tools/warnings/extra-gcc rename to tools/warnings/extra-cppflags diff --git a/tools/warnings/more-g++ b/tools/warnings/more-cflags similarity index 100% rename from tools/warnings/more-g++ rename to tools/warnings/more-cflags diff --git a/tools/warnings/more-gcc b/tools/warnings/more-cppflags similarity index 100% rename from tools/warnings/more-gcc rename to tools/warnings/more-cppflags diff --git a/tools/warnings/none-gcc b/tools/warnings/none-cflags similarity index 100% rename from tools/warnings/none-gcc rename to tools/warnings/none-cflags diff --git a/tools/warnings/none-g++ b/tools/warnings/none-cppflags similarity index 100% rename from tools/warnings/none-g++ rename to tools/warnings/none-cppflags