From 57cbc3484060f646deb0f4f652abcca4732b3458 Mon Sep 17 00:00:00 2001 From: Olivier Lenoir Date: Fri, 1 Mar 2024 12:18:35 +0100 Subject: [PATCH 01/88] mip: Add support to mip install from GitLab. Modify _rewrite_url() to allow mip install from `gitlab:` repository. Signed-off-by: Olivier Lenoir --- micropython/mip/mip/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/micropython/mip/mip/__init__.py b/micropython/mip/mip/__init__.py index 68daf32fe..0c3c6f204 100644 --- a/micropython/mip/mip/__init__.py +++ b/micropython/mip/mip/__init__.py @@ -73,6 +73,18 @@ def _rewrite_url(http://github.com/url,%20branch=None): + "/" + "/".join(url[2:]) ) + elif url.startswith("gitlab:"): + url = url[7:].split("/") + url = ( + "https://gitlab.com/" + + url[0] + + "/" + + url[1] + + "/-/raw/" + + branch + + "/" + + "/".join(url[2:]) + ) return url @@ -128,6 +140,7 @@ def _install_package(package, index, target, version, mpy): package.startswith("http://") or package.startswith("https://") or package.startswith("github:") + or package.startswith("gitlab:") ): if package.endswith(".py") or package.endswith(".mpy"): print("Downloading {} to {}".format(package, target)) From 7206da4645ded27acf1ad9e9ecfdf080c81ccb05 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 15 May 2024 13:53:01 +1000 Subject: [PATCH 02/88] mip: Bump minor version. The previous commit added a new feature (ability to install from GitLab). Signed-off-by: Damien George --- micropython/mip/manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropython/mip/manifest.py b/micropython/mip/manifest.py index 00efa5454..88fb08da1 100644 --- a/micropython/mip/manifest.py +++ b/micropython/mip/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0", description="On-device package installer for network-capable boards") +metadata(version="0.3.0", description="On-device package installer for network-capable boards") require("requests") From a2e4efa09a4c0b709b227d689e878029d5c83d0c Mon Sep 17 00:00:00 2001 From: Matt Trentini Date: Tue, 16 Apr 2024 13:43:20 +1000 Subject: [PATCH 03/88] collections: Remove micropython-lib Python implementation of deque. It's no longer necessary since the built-in C version of this type now implements all the functionality here. Signed-off-by: Matt Trentini --- .../collections-deque/collections/deque.py | 36 ------------------- python-stdlib/collections-deque/manifest.py | 4 --- .../collections/collections/__init__.py | 4 --- python-stdlib/collections/manifest.py | 2 +- 4 files changed, 1 insertion(+), 45 deletions(-) delete mode 100644 python-stdlib/collections-deque/collections/deque.py delete mode 100644 python-stdlib/collections-deque/manifest.py diff --git a/python-stdlib/collections-deque/collections/deque.py b/python-stdlib/collections-deque/collections/deque.py deleted file mode 100644 index 1d8c62d4b..000000000 --- a/python-stdlib/collections-deque/collections/deque.py +++ /dev/null @@ -1,36 +0,0 @@ -class deque: - def __init__(self, iterable=None): - if iterable is None: - self.q = [] - else: - self.q = list(iterable) - - def popleft(self): - return self.q.pop(0) - - def popright(self): - return self.q.pop() - - def pop(self): - return self.q.pop() - - def append(self, a): - self.q.append(a) - - def appendleft(self, a): - self.q.insert(0, a) - - def extend(self, a): - self.q.extend(a) - - def __len__(self): - return len(self.q) - - def __bool__(self): - return bool(self.q) - - def __iter__(self): - yield from self.q - - def __str__(self): - return "deque({})".format(self.q) diff --git a/python-stdlib/collections-deque/manifest.py b/python-stdlib/collections-deque/manifest.py deleted file mode 100644 index 0133d2bad..000000000 --- a/python-stdlib/collections-deque/manifest.py +++ /dev/null @@ -1,4 +0,0 @@ -metadata(version="0.1.3") - -require("collections") -package("collections") diff --git a/python-stdlib/collections/collections/__init__.py b/python-stdlib/collections/collections/__init__.py index 7f3be5673..36dfc1c41 100644 --- a/python-stdlib/collections/collections/__init__.py +++ b/python-stdlib/collections/collections/__init__.py @@ -6,10 +6,6 @@ from .defaultdict import defaultdict except ImportError: pass -try: - from .deque import deque -except ImportError: - pass class MutableMapping: diff --git a/python-stdlib/collections/manifest.py b/python-stdlib/collections/manifest.py index d5ef69472..0ce56d1fa 100644 --- a/python-stdlib/collections/manifest.py +++ b/python-stdlib/collections/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.2") +metadata(version="0.2.0") package("collections") From cb281a417726e35b33eb5304ba1ac86d979bcfbc Mon Sep 17 00:00:00 2001 From: Jon Foster Date: Sat, 23 Mar 2024 17:55:45 +0000 Subject: [PATCH 04/88] ntptime: Fix Year 2036 bug. Fix NTP client - it would report the wrong time after 7 Feb 2036. Signed-off-by: Jon Foster --- micropython/net/ntptime/manifest.py | 2 +- micropython/net/ntptime/ntptime.py | 27 ++++++++++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/micropython/net/ntptime/manifest.py b/micropython/net/ntptime/manifest.py index 97e3c14a3..15f832966 100644 --- a/micropython/net/ntptime/manifest.py +++ b/micropython/net/ntptime/manifest.py @@ -1,3 +1,3 @@ -metadata(description="NTP client.", version="0.1.0") +metadata(description="NTP client.", version="0.1.1") module("ntptime.py", opt=3) diff --git a/micropython/net/ntptime/ntptime.py b/micropython/net/ntptime/ntptime.py index ff0d9d202..25cc62ad1 100644 --- a/micropython/net/ntptime/ntptime.py +++ b/micropython/net/ntptime/ntptime.py @@ -22,12 +22,37 @@ def time(): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: s.settimeout(timeout) - res = s.sendto(NTP_QUERY, addr) + s.sendto(NTP_QUERY, addr) msg = s.recv(48) finally: s.close() val = struct.unpack("!I", msg[40:44])[0] + # 2024-01-01 00:00:00 converted to an NTP timestamp + MIN_NTP_TIMESTAMP = 3913056000 + + # Y2036 fix + # + # The NTP timestamp has a 32-bit count of seconds, which will wrap back + # to zero on 7 Feb 2036 at 06:28:16. + # + # We know that this software was written during 2024 (or later). + # So we know that timestamps less than MIN_NTP_TIMESTAMP are impossible. + # So if the timestamp is less than MIN_NTP_TIMESTAMP, that probably means + # that the NTP time wrapped at 2^32 seconds. (Or someone set the wrong + # time on their NTP server, but we can't really do anything about that). + # + # So in that case, we need to add in those extra 2^32 seconds, to get the + # correct timestamp. + # + # This means that this code will work until the year 2160. More precisely, + # this code will not work after 7th Feb 2160 at 06:28:15. + # + if val < MIN_NTP_TIMESTAMP: + val += 0x100000000 + + # Convert timestamp from NTP format to our internal format + EPOCH_YEAR = utime.gmtime(0)[0] if EPOCH_YEAR == 2000: # (date(2000, 1, 1) - date(1900, 1, 1)).days * 24*60*60 From 6c6fab1db1212b80293887be810ae6466fa69fa8 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 14 May 2024 15:05:35 +1000 Subject: [PATCH 05/88] all: Enable ruff F841 'Local variable is assigned to but never used'. Most of these look like they were used for print debugging and then kept in when the print statements were removed or commented. Some look like missing or incomplete functionality, these have been marked with comments where possible. Signed-off-by: Angus Gratton --- .../aioble/examples/l2cap_file_client.py | 4 +-- .../aioble/examples/l2cap_file_server.py | 5 +--- .../aioble/multitests/ble_descriptor.py | 4 +-- micropython/espflash/espflash.py | 1 - micropython/ucontextlib/tests.py | 2 +- micropython/udnspkt/udnspkt.py | 26 +++++++------------ .../urllib.urequest/urllib/urequest.py | 6 ++--- pyproject.toml | 1 - python-ecosys/aiohttp/aiohttp/__init__.py | 1 - python-ecosys/cbor2/cbor2/_decoder.py | 2 +- 10 files changed, 18 insertions(+), 34 deletions(-) diff --git a/micropython/bluetooth/aioble/examples/l2cap_file_client.py b/micropython/bluetooth/aioble/examples/l2cap_file_client.py index 68770f043..54b357d4f 100644 --- a/micropython/bluetooth/aioble/examples/l2cap_file_client.py +++ b/micropython/bluetooth/aioble/examples/l2cap_file_client.py @@ -85,7 +85,7 @@ async def size(self, path): async def download(self, path, dest): size = await self.size(path) - send_seq = await self._command(_COMMAND_SEND, path.encode()) + await self._command(_COMMAND_SEND, path.encode()) with open(dest, "wb") as f: # noqa: ASYNC101 total = 0 @@ -97,7 +97,7 @@ async def download(self, path, dest): total += n async def list(self, path): - send_seq = await self._command(_COMMAND_LIST, path.encode()) + await self._command(_COMMAND_LIST, path.encode()) results = bytearray() buf = bytearray(self._channel.our_mtu) mv = memoryview(buf) diff --git a/micropython/bluetooth/aioble/examples/l2cap_file_server.py b/micropython/bluetooth/aioble/examples/l2cap_file_server.py index c3730ffd0..c6aef6587 100644 --- a/micropython/bluetooth/aioble/examples/l2cap_file_server.py +++ b/micropython/bluetooth/aioble/examples/l2cap_file_server.py @@ -132,15 +132,12 @@ async def control_task(connection): file = msg[2:].decode() if command == _COMMAND_SEND: - op_seq = seq send_file = file l2cap_event.set() elif command == _COMMAND_RECV: - op_seq = seq recv_file = file l2cap_event.set() elif command == _COMMAND_LIST: - op_seq = seq list_path = file l2cap_event.set() elif command == _COMMAND_SIZE: @@ -148,7 +145,7 @@ async def control_task(connection): stat = os.stat(file) size = stat[6] status = 0 - except OSError as e: + except OSError: size = 0 status = _STATUS_NOT_FOUND control_characteristic.notify( diff --git a/micropython/bluetooth/aioble/multitests/ble_descriptor.py b/micropython/bluetooth/aioble/multitests/ble_descriptor.py index 2354dff6b..c45f1413c 100644 --- a/micropython/bluetooth/aioble/multitests/ble_descriptor.py +++ b/micropython/bluetooth/aioble/multitests/ble_descriptor.py @@ -32,9 +32,7 @@ async def instance0_task(): char1_desc1.write("char1_desc1") char1_desc2 = aioble.Descriptor(char1, CHAR1_DESC2_UUID, read=True, write=True) char1_desc2.write("char1_desc2") - char2 = aioble.Characteristic( - service, CHAR2_UUID, read=True, write=True, notify=True, indicate=True - ) + aioble.Characteristic(service, CHAR2_UUID, read=True, write=True, notify=True, indicate=True) char3 = aioble.Characteristic( service, CHAR3_UUID, read=True, write=True, notify=True, indicate=True ) diff --git a/micropython/espflash/espflash.py b/micropython/espflash/espflash.py index cc025836c..6d9583405 100644 --- a/micropython/espflash/espflash.py +++ b/micropython/espflash/espflash.py @@ -258,7 +258,6 @@ def flash_write_file(self, path, blksize=0x1000): print(f"Flash write size: {size} total_blocks: {total_blocks} block size: {blksize}") with open(path, "rb") as f: seq = 0 - subseq = 0 for i in range(total_blocks): buf = f.read(blksize) # Update digest diff --git a/micropython/ucontextlib/tests.py b/micropython/ucontextlib/tests.py index 4fd026ae7..163175d82 100644 --- a/micropython/ucontextlib/tests.py +++ b/micropython/ucontextlib/tests.py @@ -24,7 +24,7 @@ def test_context_manager(self): def test_context_manager_on_error(self): exc = Exception() try: - with self._manager(123) as x: + with self._manager(123): raise exc except Exception as e: self.assertEqual(exc, e) diff --git a/micropython/udnspkt/udnspkt.py b/micropython/udnspkt/udnspkt.py index e55285975..2cb11ab92 100644 --- a/micropython/udnspkt/udnspkt.py +++ b/micropython/udnspkt/udnspkt.py @@ -43,36 +43,28 @@ def parse_resp(buf, is_ipv6): if is_ipv6: typ = 28 # AAAA - id = buf.readbin(">H") + buf.readbin(">H") # id flags = buf.readbin(">H") assert flags & 0x8000 - qcnt = buf.readbin(">H") + buf.readbin(">H") # qcnt acnt = buf.readbin(">H") - nscnt = buf.readbin(">H") - addcnt = buf.readbin(">H") - # print(qcnt, acnt, nscnt, addcnt) + buf.readbin(">H") # nscnt + buf.readbin(">H") # addcnt skip_fqdn(buf) - v = buf.readbin(">H") - # print(v) - v = buf.readbin(">H") - # print(v) + buf.readbin(">H") + buf.readbin(">H") for i in range(acnt): # print("Resp #%d" % i) # v = read_fqdn(buf) # print(v) skip_fqdn(buf) - t = buf.readbin(">H") - # print("Type", t) - v = buf.readbin(">H") - # print("Class", v) - v = buf.readbin(">I") - # print("TTL", v) + t = buf.readbin(">H") # Type + buf.readbin(">H") # Class + buf.readbin(">I") # TTL rlen = buf.readbin(">H") - # print("rlen", rlen) rval = buf.read(rlen) - # print(rval) if t == typ: return rval diff --git a/micropython/urllib.urequest/urllib/urequest.py b/micropython/urllib.urequest/urllib/urequest.py index 2eff43c36..5154c0f05 100644 --- a/micropython/urllib.urequest/urllib/urequest.py +++ b/micropython/urllib.urequest/urllib/urequest.py @@ -48,10 +48,10 @@ def urlopen(url, data=None, method="GET"): if data: s.write(data) - l = s.readline() - l = l.split(None, 2) + l = s.readline() # Status-Line + # l = l.split(None, 2) # print(l) - status = int(l[1]) + # status = int(l[1]) # FIXME: Status-Code element is not currently checked while True: l = s.readline() if not l or l == b"\r\n": diff --git a/pyproject.toml b/pyproject.toml index 3b2524545..15bb05375 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,6 @@ ignore = [ "F405", "E501", "F541", - "F841", "ISC001", "ISC003", # micropython does not support implicit concatenation of f-strings "PIE810", # micropython does not support passing tuples to .startswith or .endswith diff --git a/python-ecosys/aiohttp/aiohttp/__init__.py b/python-ecosys/aiohttp/aiohttp/__init__.py index 23d227a6f..79a676de7 100644 --- a/python-ecosys/aiohttp/aiohttp/__init__.py +++ b/python-ecosys/aiohttp/aiohttp/__init__.py @@ -104,7 +104,6 @@ async def __aexit__(self, *args): async def _request(self, method, url, data=None, json=None, ssl=None, params=None, headers={}): redir_cnt = 0 - redir_url = None while redir_cnt < 2: reader = await self.request_raw(method, url, data, json, ssl, params, headers) _headers = [] diff --git a/python-ecosys/cbor2/cbor2/_decoder.py b/python-ecosys/cbor2/cbor2/_decoder.py index e38f078f3..5d509a535 100644 --- a/python-ecosys/cbor2/cbor2/_decoder.py +++ b/python-ecosys/cbor2/cbor2/_decoder.py @@ -159,7 +159,7 @@ def decode_simple_value(decoder): def decode_float16(decoder): - payload = decoder.read(2) + decoder.read(2) raise NotImplementedError # no float16 unpack function From 992eecfed416b042ba5ef80c0b0bf2ca3887549f Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 14 May 2024 15:47:26 +1000 Subject: [PATCH 06/88] all: Enable Ruff lint F541 'f-string without any placeholders'. Signed-off-by: Angus Gratton --- micropython/espflash/espflash.py | 6 +++--- micropython/lora/lora-async/lora/async_modem.py | 2 +- pyproject.toml | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/micropython/espflash/espflash.py b/micropython/espflash/espflash.py index 6d9583405..74988777a 100644 --- a/micropython/espflash/espflash.py +++ b/micropython/espflash/espflash.py @@ -235,7 +235,7 @@ def flash_read_size(self): def flash_attach(self): self._command(_CMD_SPI_ATTACH, struct.pack(" Date: Tue, 14 May 2024 15:38:47 +1000 Subject: [PATCH 07/88] all: Enable ruff E401 and E402 import lints. Mostly small cleanups to put each top-level import on its own line. But explicitly disable the lint for examples/tests which insert the current directory into the path before importing. Signed-off-by: Angus Gratton --- .../bluetooth/aioble/examples/l2cap_file_client.py | 1 + .../bluetooth/aioble/examples/l2cap_file_server.py | 1 + micropython/bluetooth/aioble/examples/temp_client.py | 1 + micropython/bluetooth/aioble/examples/temp_sensor.py | 1 + .../aioble/multitests/ble_buffered_characteristic.py | 4 +++- .../bluetooth/aioble/multitests/ble_characteristic.py | 4 +++- micropython/bluetooth/aioble/multitests/ble_descriptor.py | 4 +++- micropython/bluetooth/aioble/multitests/ble_notify.py | 4 +++- micropython/bluetooth/aioble/multitests/ble_shutdown.py | 4 +++- .../bluetooth/aioble/multitests/ble_write_capture.py | 4 +++- .../bluetooth/aioble/multitests/ble_write_order.py | 4 +++- .../bluetooth/aioble/multitests/perf_gatt_notify.py | 4 +++- micropython/bluetooth/aioble/multitests/perf_l2cap.py | 4 +++- micropython/drivers/display/lcd160cr/lcd160cr.py | 3 ++- micropython/drivers/display/lcd160cr/lcd160cr_test.py | 5 ++++- micropython/drivers/storage/sdcard/sdtest.py | 4 +++- pyproject.toml | 8 +++----- python-ecosys/aiohttp/aiohttp/__init__.py | 3 ++- python-ecosys/aiohttp/examples/client.py | 1 + python-ecosys/aiohttp/examples/compression.py | 1 + python-ecosys/aiohttp/examples/get.py | 1 + python-ecosys/aiohttp/examples/headers.py | 1 + python-ecosys/aiohttp/examples/methods.py | 1 + python-ecosys/aiohttp/examples/params.py | 1 + python-ecosys/aiohttp/examples/ws.py | 1 + python-ecosys/aiohttp/examples/ws_repl_echo.py | 1 + python-ecosys/iperf3/iperf3.py | 7 +++++-- 27 files changed, 58 insertions(+), 20 deletions(-) diff --git a/micropython/bluetooth/aioble/examples/l2cap_file_client.py b/micropython/bluetooth/aioble/examples/l2cap_file_client.py index 54b357d4f..2a75bc308 100644 --- a/micropython/bluetooth/aioble/examples/l2cap_file_client.py +++ b/micropython/bluetooth/aioble/examples/l2cap_file_client.py @@ -5,6 +5,7 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const diff --git a/micropython/bluetooth/aioble/examples/l2cap_file_server.py b/micropython/bluetooth/aioble/examples/l2cap_file_server.py index c6aef6587..0c45bd1ff 100644 --- a/micropython/bluetooth/aioble/examples/l2cap_file_server.py +++ b/micropython/bluetooth/aioble/examples/l2cap_file_server.py @@ -16,6 +16,7 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const diff --git a/micropython/bluetooth/aioble/examples/temp_client.py b/micropython/bluetooth/aioble/examples/temp_client.py index ceb1d0465..56715838c 100644 --- a/micropython/bluetooth/aioble/examples/temp_client.py +++ b/micropython/bluetooth/aioble/examples/temp_client.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const diff --git a/micropython/bluetooth/aioble/examples/temp_sensor.py b/micropython/bluetooth/aioble/examples/temp_sensor.py index 46cb966e4..bdd8c1c5e 100644 --- a/micropython/bluetooth/aioble/examples/temp_sensor.py +++ b/micropython/bluetooth/aioble/examples/temp_sensor.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const diff --git a/micropython/bluetooth/aioble/multitests/ble_buffered_characteristic.py b/micropython/bluetooth/aioble/multitests/ble_buffered_characteristic.py index 18ce7da64..91307908f 100644 --- a/micropython/bluetooth/aioble/multitests/ble_buffered_characteristic.py +++ b/micropython/bluetooth/aioble/multitests/ble_buffered_characteristic.py @@ -2,10 +2,12 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const -import time, machine +import machine +import time import uasyncio as asyncio import aioble diff --git a/micropython/bluetooth/aioble/multitests/ble_characteristic.py b/micropython/bluetooth/aioble/multitests/ble_characteristic.py index 145cf5f80..b5d1df2fe 100644 --- a/micropython/bluetooth/aioble/multitests/ble_characteristic.py +++ b/micropython/bluetooth/aioble/multitests/ble_characteristic.py @@ -2,10 +2,12 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const -import time, machine +import machine +import time import uasyncio as asyncio import aioble diff --git a/micropython/bluetooth/aioble/multitests/ble_descriptor.py b/micropython/bluetooth/aioble/multitests/ble_descriptor.py index c45f1413c..888222ff5 100644 --- a/micropython/bluetooth/aioble/multitests/ble_descriptor.py +++ b/micropython/bluetooth/aioble/multitests/ble_descriptor.py @@ -2,10 +2,12 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const -import time, machine +import machine +import time import uasyncio as asyncio import aioble diff --git a/micropython/bluetooth/aioble/multitests/ble_notify.py b/micropython/bluetooth/aioble/multitests/ble_notify.py index be2779e40..200e784c2 100644 --- a/micropython/bluetooth/aioble/multitests/ble_notify.py +++ b/micropython/bluetooth/aioble/multitests/ble_notify.py @@ -2,10 +2,12 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const -import time, machine +import machine +import time import uasyncio as asyncio import aioble diff --git a/micropython/bluetooth/aioble/multitests/ble_shutdown.py b/micropython/bluetooth/aioble/multitests/ble_shutdown.py index e7ab58570..dea915bfb 100644 --- a/micropython/bluetooth/aioble/multitests/ble_shutdown.py +++ b/micropython/bluetooth/aioble/multitests/ble_shutdown.py @@ -2,10 +2,12 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const -import time, machine +import machine +import time import uasyncio as asyncio import aioble diff --git a/micropython/bluetooth/aioble/multitests/ble_write_capture.py b/micropython/bluetooth/aioble/multitests/ble_write_capture.py index 96291a196..23c6d4422 100644 --- a/micropython/bluetooth/aioble/multitests/ble_write_capture.py +++ b/micropython/bluetooth/aioble/multitests/ble_write_capture.py @@ -2,10 +2,12 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const -import time, machine +import machine +import time import uasyncio as asyncio import aioble diff --git a/micropython/bluetooth/aioble/multitests/ble_write_order.py b/micropython/bluetooth/aioble/multitests/ble_write_order.py index 8bd15e400..10b44bca1 100644 --- a/micropython/bluetooth/aioble/multitests/ble_write_order.py +++ b/micropython/bluetooth/aioble/multitests/ble_write_order.py @@ -2,10 +2,12 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const -import time, machine +import machine +import time import uasyncio as asyncio import aioble diff --git a/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py b/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py index 193ac69d3..d601a0ee2 100644 --- a/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py +++ b/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py @@ -2,10 +2,12 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const -import time, machine +import machine +import time import uasyncio as asyncio import aioble diff --git a/micropython/bluetooth/aioble/multitests/perf_l2cap.py b/micropython/bluetooth/aioble/multitests/perf_l2cap.py index 32810d15b..9ccf54262 100644 --- a/micropython/bluetooth/aioble/multitests/perf_l2cap.py +++ b/micropython/bluetooth/aioble/multitests/perf_l2cap.py @@ -1,9 +1,11 @@ import sys +# ruff: noqa: E402 sys.path.append("") from micropython import const -import time, machine +import machine +import time import uasyncio as asyncio import aioble diff --git a/micropython/drivers/display/lcd160cr/lcd160cr.py b/micropython/drivers/display/lcd160cr/lcd160cr.py index f792418aa..b86cbff0d 100644 --- a/micropython/drivers/display/lcd160cr/lcd160cr.py +++ b/micropython/drivers/display/lcd160cr/lcd160cr.py @@ -2,9 +2,10 @@ # MIT license; Copyright (c) 2017 Damien P. George from micropython import const +import machine from utime import sleep_ms from ustruct import calcsize, pack_into -import uerrno, machine +import uerrno # for set_orient PORTRAIT = const(0) diff --git a/micropython/drivers/display/lcd160cr/lcd160cr_test.py b/micropython/drivers/display/lcd160cr/lcd160cr_test.py index c717a3fd5..b2a9c0c6a 100644 --- a/micropython/drivers/display/lcd160cr/lcd160cr_test.py +++ b/micropython/drivers/display/lcd160cr/lcd160cr_test.py @@ -1,7 +1,10 @@ # Driver test for official MicroPython LCD160CR display # MIT license; Copyright (c) 2017 Damien P. George -import time, math, framebuf, lcd160cr +import framebuf +import lcd160cr +import math +import time def get_lcd(lcd): diff --git a/micropython/drivers/storage/sdcard/sdtest.py b/micropython/drivers/storage/sdcard/sdtest.py index 018ef7c64..ce700e2a8 100644 --- a/micropython/drivers/storage/sdcard/sdtest.py +++ b/micropython/drivers/storage/sdcard/sdtest.py @@ -1,6 +1,8 @@ # Test for sdcard block protocol # Peter hinch 30th Jan 2016 -import os, sdcard, machine +import machine +import os +import sdcard def sdtest(): diff --git a/pyproject.toml b/pyproject.toml index 1c4d781ad..4776ddfe9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,14 +54,12 @@ select = [ # "UP", # pyupgrade ] ignore = [ - "E401", - "E402", "E722", - "E741", + "E741", # 'l' is currently widely used "F401", "F403", "F405", - "E501", + "E501", # line length, recommended to disable "ISC001", "ISC003", # micropython does not support implicit concatenation of f-strings "PIE810", # micropython does not support passing tuples to .startswith or .endswith @@ -74,7 +72,7 @@ ignore = [ "PLW2901", "RUF012", "RUF100", - "W191", + "W191", # tab-indent, redundant when using formatter ] line-length = 99 target-version = "py37" diff --git a/python-ecosys/aiohttp/aiohttp/__init__.py b/python-ecosys/aiohttp/aiohttp/__init__.py index 79a676de7..1565163c4 100644 --- a/python-ecosys/aiohttp/aiohttp/__init__.py +++ b/python-ecosys/aiohttp/aiohttp/__init__.py @@ -22,7 +22,8 @@ def _decode(self, data): c_encoding = self.headers.get("Content-Encoding") if c_encoding in ("gzip", "deflate", "gzip,deflate"): try: - import deflate, io + import deflate + import io if c_encoding == "deflate": with deflate.DeflateIO(io.BytesIO(data), deflate.ZLIB) as d: diff --git a/python-ecosys/aiohttp/examples/client.py b/python-ecosys/aiohttp/examples/client.py index 471737b26..0a6476064 100644 --- a/python-ecosys/aiohttp/examples/client.py +++ b/python-ecosys/aiohttp/examples/client.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/compression.py b/python-ecosys/aiohttp/examples/compression.py index 21f9cf7fd..a1c6276b2 100644 --- a/python-ecosys/aiohttp/examples/compression.py +++ b/python-ecosys/aiohttp/examples/compression.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/get.py b/python-ecosys/aiohttp/examples/get.py index 43507a6e7..087d6fb51 100644 --- a/python-ecosys/aiohttp/examples/get.py +++ b/python-ecosys/aiohttp/examples/get.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/headers.py b/python-ecosys/aiohttp/examples/headers.py index c3a92fc49..ec5c00a80 100644 --- a/python-ecosys/aiohttp/examples/headers.py +++ b/python-ecosys/aiohttp/examples/headers.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/methods.py b/python-ecosys/aiohttp/examples/methods.py index 118777c4e..af38ff652 100644 --- a/python-ecosys/aiohttp/examples/methods.py +++ b/python-ecosys/aiohttp/examples/methods.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/params.py b/python-ecosys/aiohttp/examples/params.py index 8c47e2097..9aecb2ab8 100644 --- a/python-ecosys/aiohttp/examples/params.py +++ b/python-ecosys/aiohttp/examples/params.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/ws.py b/python-ecosys/aiohttp/examples/ws.py index e989a39c5..b96ee6819 100644 --- a/python-ecosys/aiohttp/examples/ws.py +++ b/python-ecosys/aiohttp/examples/ws.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/aiohttp/examples/ws_repl_echo.py b/python-ecosys/aiohttp/examples/ws_repl_echo.py index 9393620e3..c41a4ee5e 100644 --- a/python-ecosys/aiohttp/examples/ws_repl_echo.py +++ b/python-ecosys/aiohttp/examples/ws_repl_echo.py @@ -1,5 +1,6 @@ import sys +# ruff: noqa: E402 sys.path.insert(0, ".") import aiohttp import asyncio diff --git a/python-ecosys/iperf3/iperf3.py b/python-ecosys/iperf3/iperf3.py index a5c54445d..363d10d59 100644 --- a/python-ecosys/iperf3/iperf3.py +++ b/python-ecosys/iperf3/iperf3.py @@ -12,9 +12,12 @@ iperf3.client('192.168.1.5', udp=True, reverse=True) """ -import sys, struct -import time, select, socket import json +import select +import socket +import struct +import sys +import time # Provide a urandom() function, supporting devices without os.urandom(). try: From 2b0d7610cef301881fea2c1e8994227367196093 Mon Sep 17 00:00:00 2001 From: AuroraTea <1352685369@qq.com> Date: Sun, 14 Apr 2024 16:35:52 +0800 Subject: [PATCH 08/88] aiohttp: Fix type of header's Sec-WebSocket-Key. The function `binascii.b2a_base64()` returns a `bytes`, but here needs a string. Otherwise, the value of `Sec-WebSocket-Key` in the headers will be `b''`. Signed-off-by: AuroraTea <1352685369@qq.com> --- python-ecosys/aiohttp/aiohttp/aiohttp_ws.py | 4 ++-- python-ecosys/aiohttp/manifest.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py b/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py index e5575a11c..07d833730 100644 --- a/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py +++ b/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py @@ -86,7 +86,7 @@ def _parse_frame_header(cls, header): def _process_websocket_frame(self, opcode, payload): if opcode == self.TEXT: - payload = payload.decode() + payload = str(payload, "utf-8") elif opcode == self.BINARY: pass elif opcode == self.CLOSE: @@ -143,7 +143,7 @@ async def handshake(self, uri, ssl, req): headers["Host"] = f"{uri.hostname}:{uri.port}" headers["Connection"] = "Upgrade" headers["Upgrade"] = "websocket" - headers["Sec-WebSocket-Key"] = key + headers["Sec-WebSocket-Key"] = str(key, "utf-8") headers["Sec-WebSocket-Version"] = "13" headers["Origin"] = f"{_http_proto}://{uri.hostname}:{uri.port}" diff --git a/python-ecosys/aiohttp/manifest.py b/python-ecosys/aiohttp/manifest.py index 9cb2ef50f..748970e5b 100644 --- a/python-ecosys/aiohttp/manifest.py +++ b/python-ecosys/aiohttp/manifest.py @@ -1,6 +1,6 @@ metadata( description="HTTP client module for MicroPython asyncio module", - version="0.0.2", + version="0.0.3", pypi="aiohttp", ) From 191494ede7fb11585b2cba418f5eeee8d3d3aab0 Mon Sep 17 00:00:00 2001 From: Stephen More Date: Mon, 25 Mar 2024 08:49:00 -0400 Subject: [PATCH 09/88] aioble/examples/temp_sensor.py: Properly notify on update. This ensures that the peripheral notifies subscribed clients when the characteristic is written to. Signed-off-by: Stephen More --- micropython/bluetooth/aioble/examples/temp_sensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropython/bluetooth/aioble/examples/temp_sensor.py b/micropython/bluetooth/aioble/examples/temp_sensor.py index bdd8c1c5e..8ab3df700 100644 --- a/micropython/bluetooth/aioble/examples/temp_sensor.py +++ b/micropython/bluetooth/aioble/examples/temp_sensor.py @@ -40,7 +40,7 @@ def _encode_temperature(temp_deg_c): async def sensor_task(): t = 24.5 while True: - temp_characteristic.write(_encode_temperature(t)) + temp_characteristic.write(_encode_temperature(t), send_update=True) t += random.uniform(-0.5, 0.5) await asyncio.sleep_ms(1000) From d4362d5cc3a6d90fabc9a684e1e7c5a29cc47f6e Mon Sep 17 00:00:00 2001 From: Stephen More Date: Mon, 25 Mar 2024 16:10:09 -0400 Subject: [PATCH 10/88] aioble/examples/temp_sensor.py: Wait forever for client to disconnect. This sets the disconnected timeout to None, so that the peripheral waits forever for the client to disconnect. Previously the peripheral would abort the connection after 60 seconds (because that's the default timeout). Signed-off-by: Stephen More --- micropython/bluetooth/aioble/examples/temp_sensor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropython/bluetooth/aioble/examples/temp_sensor.py b/micropython/bluetooth/aioble/examples/temp_sensor.py index 8ab3df700..29f774bee 100644 --- a/micropython/bluetooth/aioble/examples/temp_sensor.py +++ b/micropython/bluetooth/aioble/examples/temp_sensor.py @@ -56,7 +56,7 @@ async def peripheral_task(): appearance=_ADV_APPEARANCE_GENERIC_THERMOMETER, ) as connection: print("Connection from", connection.device) - await connection.disconnected() + await connection.disconnected(timeout_ms=None) # Run both tasks. From da46c4b9f7b4dd590c8223ee860d33d28c965e79 Mon Sep 17 00:00:00 2001 From: Brian Pugh Date: Fri, 19 May 2023 09:10:09 -0700 Subject: [PATCH 11/88] pathlib: Add __rtruediv__ magic method to pathlib.Path. MicroPython now supports this behaviour of __rtruediv__. --- python-stdlib/pathlib/pathlib.py | 3 +++ python-stdlib/pathlib/tests/test_pathlib.py | 11 +++++++++++ 2 files changed, 14 insertions(+) diff --git a/python-stdlib/pathlib/pathlib.py b/python-stdlib/pathlib/pathlib.py index d01d81d32..e0f961373 100644 --- a/python-stdlib/pathlib/pathlib.py +++ b/python-stdlib/pathlib/pathlib.py @@ -47,6 +47,9 @@ def __init__(self, *segments): def __truediv__(self, other): return Path(self._path, str(other)) + def __rtruediv__(self, other): + return Path(other, self._path) + def __repr__(self): return f'{type(self).__name__}("{self._path}")' diff --git a/python-stdlib/pathlib/tests/test_pathlib.py b/python-stdlib/pathlib/tests/test_pathlib.py index c52cd9705..e632e1242 100644 --- a/python-stdlib/pathlib/tests/test_pathlib.py +++ b/python-stdlib/pathlib/tests/test_pathlib.py @@ -322,3 +322,14 @@ def test_with_suffix(self): self.assertTrue(Path("foo/test").with_suffix(".tar") == Path("foo/test.tar")) self.assertTrue(Path("foo/bar.bin").with_suffix(".txt") == Path("foo/bar.txt")) self.assertTrue(Path("bar.txt").with_suffix("") == Path("bar")) + + def test_rtruediv(self): + """Works as of micropython ea7031f""" + res = "foo" / Path("bar") + self.assertTrue(res == Path("foo/bar")) + + def test_rtruediv_inplace(self): + """Works as of micropython ea7031f""" + res = "foo" + res /= Path("bar") + self.assertTrue(res == Path("foo/bar")) From f0b683218efafae904db060eff48d58eaf59c142 Mon Sep 17 00:00:00 2001 From: Rob Knegjens Date: Tue, 5 Apr 2022 12:05:06 -0700 Subject: [PATCH 12/88] aioble/examples/temp_client.py: Check connection before reading temp. Only read from the temp characteristic if the connection is still active. Improves the example by avoiding a TypeError exception if/when the sensor disconnects. --- micropython/bluetooth/aioble/examples/temp_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropython/bluetooth/aioble/examples/temp_client.py b/micropython/bluetooth/aioble/examples/temp_client.py index 56715838c..42752d8c9 100644 --- a/micropython/bluetooth/aioble/examples/temp_client.py +++ b/micropython/bluetooth/aioble/examples/temp_client.py @@ -55,7 +55,7 @@ async def main(): print("Timeout discovering services/characteristics") return - while True: + while connection.is_connected(): temp_deg_c = _decode_temperature(await temp_characteristic.read()) print("Temperature: {:.2f}".format(temp_deg_c)) await asyncio.sleep_ms(1000) From e7f605df335f5d30bb1f534981aa42f95935e8d3 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Fri, 16 Sep 2022 12:11:25 +1000 Subject: [PATCH 13/88] aioble/device.py: Always create connection._event. If the client disconnects immediately after connection, the irq can be run before the initial connect handler has finished. --- micropython/bluetooth/aioble/aioble/device.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/micropython/bluetooth/aioble/aioble/device.py b/micropython/bluetooth/aioble/aioble/device.py index 265d62157..30a54a4fd 100644 --- a/micropython/bluetooth/aioble/aioble/device.py +++ b/micropython/bluetooth/aioble/aioble/device.py @@ -164,7 +164,7 @@ def __init__(self, device): # This event is fired by the IRQ both for connection and disconnection # and controls the device_task. - self._event = None + self._event = asyncio.ThreadSafeFlag() # If we're waiting for a pending MTU exchange. self._mtu_event = None @@ -207,9 +207,6 @@ async def device_task(self): t._task.cancel() def _run_task(self): - # Event will be already created this if we initiated connection. - self._event = self._event or asyncio.ThreadSafeFlag() - self._task = asyncio.create_task(self.device_task()) async def disconnect(self, timeout_ms=2000): From db7f9a18d4833275637bb53411d6671cd83bc533 Mon Sep 17 00:00:00 2001 From: Rob Knegjens Date: Wed, 6 Apr 2022 13:27:24 -0700 Subject: [PATCH 14/88] aioble/device.py: Make default timeout None for disconnected() method. The value for the `timeout_ms` optional argument to `DeviceConnection.disconnected()` async method is changed from 60000 to None. This way users awaiting a device disconnection using `await connection.disconnected()` won't be surprised by a 1 minute timeout. --- micropython/bluetooth/aioble-core/manifest.py | 2 +- micropython/bluetooth/aioble/aioble/device.py | 2 +- micropython/bluetooth/aioble/manifest.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/micropython/bluetooth/aioble-core/manifest.py b/micropython/bluetooth/aioble-core/manifest.py index 2448769e6..c2d335b5c 100644 --- a/micropython/bluetooth/aioble-core/manifest.py +++ b/micropython/bluetooth/aioble-core/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0") +metadata(version="0.3.0") package( "aioble", diff --git a/micropython/bluetooth/aioble/aioble/device.py b/micropython/bluetooth/aioble/aioble/device.py index 30a54a4fd..8844eb42a 100644 --- a/micropython/bluetooth/aioble/aioble/device.py +++ b/micropython/bluetooth/aioble/aioble/device.py @@ -212,7 +212,7 @@ def _run_task(self): async def disconnect(self, timeout_ms=2000): await self.disconnected(timeout_ms, disconnect=True) - async def disconnected(self, timeout_ms=60000, disconnect=False): + async def disconnected(self, timeout_ms=None, disconnect=False): if not self.is_connected(): return diff --git a/micropython/bluetooth/aioble/manifest.py b/micropython/bluetooth/aioble/manifest.py index 2979a726b..565d6060f 100644 --- a/micropython/bluetooth/aioble/manifest.py +++ b/micropython/bluetooth/aioble/manifest.py @@ -3,7 +3,7 @@ # code. This allows (for development purposes) all the files to live in the # one directory. -metadata(version="0.4.1") +metadata(version="0.5.0") # Default installation gives you everything. Install the individual # components (or a combination of them) if you want a more minimal install. From e5389eb26ab4a48a0414b3a7a6d83bd7fadf1abe Mon Sep 17 00:00:00 2001 From: Trent Piepho Date: Wed, 10 Jan 2024 13:40:24 -0800 Subject: [PATCH 15/88] aioble/peripheral.py: Place multiple UUIDs in single advertisement LTV. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When multiple UUIDs of the same size are advertised, they should all be listed in a single LTV. Supplement to the Bluetooth Core Specification, Part A, §1.1.1: "A packet or data block shall not contain more than one instance for each Service UUID data size." When aioble construct the advertisement data, it is creating a new data block for each UUID that contains only that single UUID. Rather than, e.g., a single 16-bit UUID block with a list of multiple UUIDs. Not only is this against the specification, it wastes two bytes of limited advertisement space per UUID beyond the first for the repeated data block length and type fields. Fix this by grouping each UUID size together. Signed-off-by: Trent Piepho --- .../bluetooth/aioble-peripheral/manifest.py | 2 +- micropython/bluetooth/aioble/aioble/peripheral.py | 15 +++++++-------- micropython/bluetooth/aioble/manifest.py | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/micropython/bluetooth/aioble-peripheral/manifest.py b/micropython/bluetooth/aioble-peripheral/manifest.py index dd4dd122d..0aec4d21e 100644 --- a/micropython/bluetooth/aioble-peripheral/manifest.py +++ b/micropython/bluetooth/aioble-peripheral/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0") +metadata(version="0.2.1") require("aioble-core") diff --git a/micropython/bluetooth/aioble/aioble/peripheral.py b/micropython/bluetooth/aioble/aioble/peripheral.py index 099f2c557..a156ccd21 100644 --- a/micropython/bluetooth/aioble/aioble/peripheral.py +++ b/micropython/bluetooth/aioble/aioble/peripheral.py @@ -129,14 +129,13 @@ async def advertise( # Services are prioritised to go in the advertising data because iOS supports # filtering scan results by service only, so services must come first. if services: - for uuid in services: - b = bytes(uuid) - if len(b) == 2: - resp_data = _append(adv_data, resp_data, _ADV_TYPE_UUID16_COMPLETE, b) - elif len(b) == 4: - resp_data = _append(adv_data, resp_data, _ADV_TYPE_UUID32_COMPLETE, b) - elif len(b) == 16: - resp_data = _append(adv_data, resp_data, _ADV_TYPE_UUID128_COMPLETE, b) + for uuid_len, code in ( + (2, _ADV_TYPE_UUID16_COMPLETE), + (4, _ADV_TYPE_UUID32_COMPLETE), + (16, _ADV_TYPE_UUID128_COMPLETE), + ): + if uuids := [bytes(uuid) for uuid in services if len(bytes(uuid)) == uuid_len]: + resp_data = _append(adv_data, resp_data, code, b"".join(uuids)) if name: resp_data = _append(adv_data, resp_data, _ADV_TYPE_NAME, name) diff --git a/micropython/bluetooth/aioble/manifest.py b/micropython/bluetooth/aioble/manifest.py index 565d6060f..8a9df1aac 100644 --- a/micropython/bluetooth/aioble/manifest.py +++ b/micropython/bluetooth/aioble/manifest.py @@ -3,7 +3,7 @@ # code. This allows (for development purposes) all the files to live in the # one directory. -metadata(version="0.5.0") +metadata(version="0.5.1") # Default installation gives you everything. Install the individual # components (or a combination of them) if you want a more minimal install. From 46e243c592ab12d905248a2138e4910e25a88ace Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 25 May 2024 17:37:41 +1000 Subject: [PATCH 16/88] aioble/central.py: Fix ScanResult.services when decoding UUIDs. Fixes are needed to support the cases of: - There may be more than one UUID per advertising field. - The UUID advertising field may be empty (no UUIDs). - Constructing 32-bit `bluetooth.UUID()` entities, which must be done by passing in a 4-byte bytes object, not an integer. Signed-off-by: Damien George --- micropython/bluetooth/aioble-central/manifest.py | 2 +- micropython/bluetooth/aioble/aioble/central.py | 14 ++++++++------ micropython/bluetooth/aioble/manifest.py | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/micropython/bluetooth/aioble-central/manifest.py b/micropython/bluetooth/aioble-central/manifest.py index 128c90642..9564ecf77 100644 --- a/micropython/bluetooth/aioble-central/manifest.py +++ b/micropython/bluetooth/aioble-central/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.1") +metadata(version="0.2.2") require("aioble-core") diff --git a/micropython/bluetooth/aioble/aioble/central.py b/micropython/bluetooth/aioble/aioble/central.py index adfc9729e..2f1492d08 100644 --- a/micropython/bluetooth/aioble/aioble/central.py +++ b/micropython/bluetooth/aioble/aioble/central.py @@ -195,12 +195,14 @@ def name(self): # Generator that enumerates the service UUIDs that are advertised. def services(self): - for u in self._decode_field(_ADV_TYPE_UUID16_INCOMPLETE, _ADV_TYPE_UUID16_COMPLETE): - yield bluetooth.UUID(struct.unpack(" Date: Sat, 25 May 2024 17:45:42 +1000 Subject: [PATCH 17/88] aioble/multitests: Add test for advertising and scanning services. This tests both encoding and decoding multiple 16-bit and 32-bit services within the one advertising field. Signed-off-by: Damien George --- .../multitests/ble_advertise_services.py | 71 +++++++++++++++++++ .../multitests/ble_advertise_services.py.exp | 8 +++ 2 files changed, 79 insertions(+) create mode 100644 micropython/bluetooth/aioble/multitests/ble_advertise_services.py create mode 100644 micropython/bluetooth/aioble/multitests/ble_advertise_services.py.exp diff --git a/micropython/bluetooth/aioble/multitests/ble_advertise_services.py b/micropython/bluetooth/aioble/multitests/ble_advertise_services.py new file mode 100644 index 000000000..d849c387f --- /dev/null +++ b/micropython/bluetooth/aioble/multitests/ble_advertise_services.py @@ -0,0 +1,71 @@ +# Test advertising multiple services, and scanning them. + +import sys + +# ruff: noqa: E402 +sys.path.append("") + +import asyncio +import aioble +import bluetooth + +TIMEOUT_MS = 5000 + +_SERVICE_16_A = bluetooth.UUID(0x180F) # Battery Service +_SERVICE_16_B = bluetooth.UUID(0x181A) # Environmental Sensing Service +_SERVICE_32_A = bluetooth.UUID("AB12") # random +_SERVICE_32_B = bluetooth.UUID("CD34") # random + + +# Acting in peripheral role (advertising). +async def instance0_task(): + multitest.globals(BDADDR=aioble.config("mac")) + multitest.next() + + # Advertise, and wait for central to connect to us. + print("advertise") + async with await aioble.advertise( + 20_000, + name="MPY", + services=[_SERVICE_16_A, _SERVICE_16_B, _SERVICE_32_A, _SERVICE_32_B], + timeout_ms=TIMEOUT_MS, + ) as connection: + print("connected") + await connection.disconnected() + print("disconnected") + + +def instance0(): + try: + asyncio.run(instance0_task()) + finally: + aioble.stop() + + +# Acting in central role (scanning). +async def instance1_task(): + multitest.next() + + wanted_device = aioble.Device(*BDADDR) + + # Scan for the wanted device/peripheral and print its advertised services. + async with aioble.scan(5000, interval_us=30000, window_us=30000, active=True) as scanner: + async for result in scanner: + if result.device == wanted_device: + services = list(result.services()) + if services: + print(services) + break + + # Connect to peripheral and then disconnect. + print("connect") + device = aioble.Device(*BDADDR) + async with await device.connect(timeout_ms=TIMEOUT_MS): + print("disconnect") + + +def instance1(): + try: + asyncio.run(instance1_task()) + finally: + aioble.stop() diff --git a/micropython/bluetooth/aioble/multitests/ble_advertise_services.py.exp b/micropython/bluetooth/aioble/multitests/ble_advertise_services.py.exp new file mode 100644 index 000000000..c0b2d974a --- /dev/null +++ b/micropython/bluetooth/aioble/multitests/ble_advertise_services.py.exp @@ -0,0 +1,8 @@ +--- instance0 --- +advertise +connected +disconnected +--- instance1 --- +[UUID(0x180f), UUID(0x181a), UUID(0x32314241), UUID(0x34334443)] +connect +disconnect From 1e792c39d3eb0f7bb2ddbf9b47eda479f32c9ae2 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 25 May 2024 18:24:19 +1000 Subject: [PATCH 18/88] aioble/multitests: Adjust expected output for write capture test. Testing shows that the first two writes always go through and the rest are dropped, so update the .exp file to match that. Signed-off-by: Damien George --- .../bluetooth/aioble/multitests/ble_write_capture.py.exp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropython/bluetooth/aioble/multitests/ble_write_capture.py.exp b/micropython/bluetooth/aioble/multitests/ble_write_capture.py.exp index dd0c6d688..366af008b 100644 --- a/micropython/bluetooth/aioble/multitests/ble_write_capture.py.exp +++ b/micropython/bluetooth/aioble/multitests/ble_write_capture.py.exp @@ -2,7 +2,7 @@ advertise connected written b'central0' -written b'central2' +written b'central1' written b'central0' True written b'central1' True written b'central2' True From 2c30a4e91bc5521a78efcd32ac04bc6ba928f4c7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 25 May 2024 18:24:38 +1000 Subject: [PATCH 19/88] aioble/multitests: Use multitest.output_metric for perf results. The perf multitests now "pass" when run. Signed-off-by: Damien George --- micropython/bluetooth/aioble/multitests/perf_gatt_notify.py | 6 +++++- .../bluetooth/aioble/multitests/perf_gatt_notify.py.exp | 4 ++++ micropython/bluetooth/aioble/multitests/perf_l2cap.py | 6 +++++- micropython/bluetooth/aioble/multitests/perf_l2cap.py.exp | 4 ++++ 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py b/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py index d601a0ee2..3d3159f59 100644 --- a/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py +++ b/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py @@ -47,6 +47,8 @@ async def instance0_task(): 20_000, adv_data=b"\x02\x01\x06\x04\xffMPY", timeout_ms=TIMEOUT_MS ) + print("connect") + client_characteristic = await discover_server(connection) # Give the central enough time to discover chars. @@ -61,7 +63,7 @@ async def instance0_task(): ticks_end = time.ticks_ms() ticks_total = time.ticks_diff(ticks_end, ticks_start) - print( + multitest.output_metric( "Acknowledged {} notifications in {} ms. {} ms/notification.".format( _NUM_NOTIFICATIONS, ticks_total, ticks_total // _NUM_NOTIFICATIONS ) @@ -87,6 +89,8 @@ async def instance1_task(): device = aioble.Device(*BDADDR) connection = await device.connect(timeout_ms=TIMEOUT_MS) + print("connect") + client_characteristic = await discover_server(connection) for i in range(_NUM_NOTIFICATIONS): diff --git a/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py.exp b/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py.exp index e69de29bb..4b7d220a0 100644 --- a/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py.exp +++ b/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py.exp @@ -0,0 +1,4 @@ +--- instance0 --- +connect +--- instance1 --- +connect diff --git a/micropython/bluetooth/aioble/multitests/perf_l2cap.py b/micropython/bluetooth/aioble/multitests/perf_l2cap.py index 9ccf54262..e21efd6fa 100644 --- a/micropython/bluetooth/aioble/multitests/perf_l2cap.py +++ b/micropython/bluetooth/aioble/multitests/perf_l2cap.py @@ -32,6 +32,8 @@ async def instance0_task(): 20_000, adv_data=b"\x02\x01\x06\x04\xffMPY", timeout_ms=TIMEOUT_MS ) + print("connect") + channel = await connection.l2cap_accept(_L2CAP_PSM, _L2CAP_MTU, timeout_ms=TIMEOUT_MS) random.seed(_RANDOM_SEED) @@ -66,6 +68,8 @@ async def instance1_task(): device = aioble.Device(*BDADDR) connection = await device.connect(timeout_ms=TIMEOUT_MS) + print("connect") + await asyncio.sleep_ms(500) channel = await connection.l2cap_connect(_L2CAP_PSM, _L2CAP_MTU, timeout_ms=TIMEOUT_MS) @@ -90,7 +94,7 @@ async def instance1_task(): ticks_end = time.ticks_ms() total_ticks = time.ticks_diff(ticks_end, ticks_first_byte) - print( + multitest.output_metric( "Received {}/{} bytes in {} ms. {} B/s".format( recv_bytes, recv_correct, total_ticks, recv_bytes * 1000 // total_ticks ) diff --git a/micropython/bluetooth/aioble/multitests/perf_l2cap.py.exp b/micropython/bluetooth/aioble/multitests/perf_l2cap.py.exp index e69de29bb..4b7d220a0 100644 --- a/micropython/bluetooth/aioble/multitests/perf_l2cap.py.exp +++ b/micropython/bluetooth/aioble/multitests/perf_l2cap.py.exp @@ -0,0 +1,4 @@ +--- instance0 --- +connect +--- instance1 --- +connect From 50ed36fbeb919753bcc26ce13a8cffd7691d06ef Mon Sep 17 00:00:00 2001 From: Damien George Date: Sat, 25 May 2024 21:01:58 +1000 Subject: [PATCH 20/88] pyusb: Add MicroPython implementation of PyUSB library. Signed-off-by: Damien George --- unix-ffi/pyusb/examples/lsusb.py | 18 +++ unix-ffi/pyusb/manifest.py | 3 + unix-ffi/pyusb/usb/__init__.py | 2 + unix-ffi/pyusb/usb/control.py | 10 ++ unix-ffi/pyusb/usb/core.py | 239 +++++++++++++++++++++++++++++++ unix-ffi/pyusb/usb/util.py | 16 +++ 6 files changed, 288 insertions(+) create mode 100644 unix-ffi/pyusb/examples/lsusb.py create mode 100644 unix-ffi/pyusb/manifest.py create mode 100644 unix-ffi/pyusb/usb/__init__.py create mode 100644 unix-ffi/pyusb/usb/control.py create mode 100644 unix-ffi/pyusb/usb/core.py create mode 100644 unix-ffi/pyusb/usb/util.py diff --git a/unix-ffi/pyusb/examples/lsusb.py b/unix-ffi/pyusb/examples/lsusb.py new file mode 100644 index 000000000..549043567 --- /dev/null +++ b/unix-ffi/pyusb/examples/lsusb.py @@ -0,0 +1,18 @@ +# Simple example to list attached USB devices. + +import usb.core + +for device in usb.core.find(find_all=True): + print("ID {:04x}:{:04x}".format(device.idVendor, device.idProduct)) + for cfg in device: + print( + " config numitf={} value={} attr={} power={}".format( + cfg.bNumInterfaces, cfg.bConfigurationValue, cfg.bmAttributes, cfg.bMaxPower + ) + ) + for itf in cfg: + print( + " interface class={} subclass={}".format( + itf.bInterfaceClass, itf.bInterfaceSubClass + ) + ) diff --git a/unix-ffi/pyusb/manifest.py b/unix-ffi/pyusb/manifest.py new file mode 100644 index 000000000..d60076255 --- /dev/null +++ b/unix-ffi/pyusb/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.0", pypi="pyusb") + +package("usb") diff --git a/unix-ffi/pyusb/usb/__init__.py b/unix-ffi/pyusb/usb/__init__.py new file mode 100644 index 000000000..19afe623c --- /dev/null +++ b/unix-ffi/pyusb/usb/__init__.py @@ -0,0 +1,2 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George diff --git a/unix-ffi/pyusb/usb/control.py b/unix-ffi/pyusb/usb/control.py new file mode 100644 index 000000000..b03a89464 --- /dev/null +++ b/unix-ffi/pyusb/usb/control.py @@ -0,0 +1,10 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George + + +def get_descriptor(dev, desc_size, desc_type, desc_index, wIndex=0): + wValue = desc_index | desc_type << 8 + d = dev.ctrl_transfer(0x80, 0x06, wValue, wIndex, desc_size) + if len(d) < 2: + raise Exception("invalid descriptor") + return d diff --git a/unix-ffi/pyusb/usb/core.py b/unix-ffi/pyusb/usb/core.py new file mode 100644 index 000000000..bfb0a028d --- /dev/null +++ b/unix-ffi/pyusb/usb/core.py @@ -0,0 +1,239 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George + +import sys +import ffi +import uctypes + +if sys.maxsize >> 32: + UINTPTR_SIZE = 8 + UINTPTR = uctypes.UINT64 +else: + UINTPTR_SIZE = 4 + UINTPTR = uctypes.UINT32 + + +def _align_word(x): + return (x + UINTPTR_SIZE - 1) & ~(UINTPTR_SIZE - 1) + + +ptr_descriptor = (0 | uctypes.ARRAY, 1 | UINTPTR) + +libusb_device_descriptor = { + "bLength": 0 | uctypes.UINT8, + "bDescriptorType": 1 | uctypes.UINT8, + "bcdUSB": 2 | uctypes.UINT16, + "bDeviceClass": 4 | uctypes.UINT8, + "bDeviceSubClass": 5 | uctypes.UINT8, + "bDeviceProtocol": 6 | uctypes.UINT8, + "bMaxPacketSize0": 7 | uctypes.UINT8, + "idVendor": 8 | uctypes.UINT16, + "idProduct": 10 | uctypes.UINT16, + "bcdDevice": 12 | uctypes.UINT16, + "iManufacturer": 14 | uctypes.UINT8, + "iProduct": 15 | uctypes.UINT8, + "iSerialNumber": 16 | uctypes.UINT8, + "bNumConfigurations": 17 | uctypes.UINT8, +} + +libusb_config_descriptor = { + "bLength": 0 | uctypes.UINT8, + "bDescriptorType": 1 | uctypes.UINT8, + "wTotalLength": 2 | uctypes.UINT16, + "bNumInterfaces": 4 | uctypes.UINT8, + "bConfigurationValue": 5 | uctypes.UINT8, + "iConfiguration": 6 | uctypes.UINT8, + "bmAttributes": 7 | uctypes.UINT8, + "MaxPower": 8 | uctypes.UINT8, + "interface": _align_word(9) | UINTPTR, # array of libusb_interface + "extra": (_align_word(9) + UINTPTR_SIZE) | UINTPTR, + "extra_length": (_align_word(9) + 2 * UINTPTR_SIZE) | uctypes.INT, +} + +libusb_interface = { + "altsetting": 0 | UINTPTR, # array of libusb_interface_descriptor + "num_altsetting": UINTPTR_SIZE | uctypes.INT, +} + +libusb_interface_descriptor = { + "bLength": 0 | uctypes.UINT8, + "bDescriptorType": 1 | uctypes.UINT8, + "bInterfaceNumber": 2 | uctypes.UINT8, + "bAlternateSetting": 3 | uctypes.UINT8, + "bNumEndpoints": 4 | uctypes.UINT8, + "bInterfaceClass": 5 | uctypes.UINT8, + "bInterfaceSubClass": 6 | uctypes.UINT8, + "bInterfaceProtocol": 7 | uctypes.UINT8, + "iInterface": 8 | uctypes.UINT8, + "endpoint": _align_word(9) | UINTPTR, + "extra": (_align_word(9) + UINTPTR_SIZE) | UINTPTR, + "extra_length": (_align_word(9) + 2 * UINTPTR_SIZE) | uctypes.INT, +} + +libusb = ffi.open("libusb-1.0.so") +libusb_init = libusb.func("i", "libusb_init", "p") +libusb_exit = libusb.func("v", "libusb_exit", "p") +libusb_get_device_list = libusb.func("i", "libusb_get_device_list", "pp") # return is ssize_t +libusb_free_device_list = libusb.func("v", "libusb_free_device_list", "pi") +libusb_get_device_descriptor = libusb.func("i", "libusb_get_device_descriptor", "pp") +libusb_get_config_descriptor = libusb.func("i", "libusb_get_config_descriptor", "pBp") +libusb_free_config_descriptor = libusb.func("v", "libusb_free_config_descriptor", "p") +libusb_open = libusb.func("i", "libusb_open", "pp") +libusb_set_configuration = libusb.func("i", "libusb_set_configuration", "pi") +libusb_claim_interface = libusb.func("i", "libusb_claim_interface", "pi") +libusb_control_transfer = libusb.func("i", "libusb_control_transfer", "pBBHHpHI") + + +def _new(sdesc): + buf = bytearray(uctypes.sizeof(sdesc)) + s = uctypes.struct(uctypes.addressof(buf), sdesc) + return s + + +class Interface: + def __init__(self, descr): + # Public attributes. + self.bInterfaceClass = descr.bInterfaceClass + self.bInterfaceSubClass = descr.bInterfaceSubClass + self.iInterface = descr.iInterface + self.extra_descriptors = uctypes.bytes_at(descr.extra, descr.extra_length) + + +class Configuration: + def __init__(self, dev, cfg_idx): + cfgs = _new(ptr_descriptor) + if libusb_get_config_descriptor(dev._dev, cfg_idx, cfgs) != 0: + libusb_exit(0) + raise Exception + descr = uctypes.struct(cfgs[0], libusb_config_descriptor) + + # Extract all needed info because descr is going to be free'd at the end. + self._itfs = [] + itf_array = uctypes.struct( + descr.interface, (0 | uctypes.ARRAY, descr.bNumInterfaces, libusb_interface) + ) + for i in range(descr.bNumInterfaces): + itf = itf_array[i] + alt_array = uctypes.struct( + itf.altsetting, + (0 | uctypes.ARRAY, itf.num_altsetting, libusb_interface_descriptor), + ) + for j in range(itf.num_altsetting): + alt = alt_array[j] + self._itfs.append(Interface(alt)) + + # Public attributes. + self.bNumInterfaces = descr.bNumInterfaces + self.bConfigurationValue = descr.bConfigurationValue + self.bmAttributes = descr.bmAttributes + self.bMaxPower = descr.MaxPower + self.extra_descriptors = uctypes.bytes_at(descr.extra, descr.extra_length) + + # Free descr memory in the driver. + libusb_free_config_descriptor(cfgs[0]) + + def __iter__(self): + return iter(self._itfs) + + +class Device: + _TIMEOUT_DEFAULT = 1000 + + def __init__(self, dev, descr): + self._dev = dev + self._num_cfg = descr.bNumConfigurations + self._handle = None + self._claim_itf = set() + + # Public attributes. + self.idVendor = descr.idVendor + self.idProduct = descr.idProduct + + def __iter__(self): + for i in range(self._num_cfg): + yield Configuration(self, i) + + def __getitem__(self, i): + return Configuration(self, i) + + def _open(self): + if self._handle is None: + # Open the USB device. + handle = _new(ptr_descriptor) + if libusb_open(self._dev, handle) != 0: + libusb_exit(0) + raise Exception + self._handle = handle[0] + + def _claim_interface(self, i): + if libusb_claim_interface(self._handle, i) != 0: + libusb_exit(0) + raise Exception + + def set_configuration(self): + # Select default configuration. + self._open() + cfg = Configuration(self, 0).bConfigurationValue + ret = libusb_set_configuration(self._handle, cfg) + if ret != 0: + libusb_exit(0) + raise Exception + + def ctrl_transfer( + self, bmRequestType, bRequest, wValue=0, wIndex=0, data_or_wLength=None, timeout=None + ): + if data_or_wLength is None: + l = 0 + data = bytes() + elif isinstance(data_or_wLength, int): + l = data_or_wLength + data = bytearray(l) + else: + l = len(data_or_wLength) + data = data_or_wLength + self._open() + if wIndex & 0xFF not in self._claim_itf: + self._claim_interface(wIndex & 0xFF) + self._claim_itf.add(wIndex & 0xFF) + if timeout is None: + timeout = self._TIMEOUT_DEFAULT + ret = libusb_control_transfer( + self._handle, bmRequestType, bRequest, wValue, wIndex, data, l, timeout * 1000 + ) + if ret < 0: + libusb_exit(0) + raise Exception + if isinstance(data_or_wLength, int): + return data + else: + return ret + + +def find(*, find_all=False, custom_match=None, idVendor=None, idProduct=None): + if libusb_init(0) < 0: + raise Exception + + devs = _new(ptr_descriptor) + count = libusb_get_device_list(0, devs) + if count < 0: + libusb_exit(0) + raise Exception + + dev_array = uctypes.struct(devs[0], (0 | uctypes.ARRAY, count | UINTPTR)) + descr = _new(libusb_device_descriptor) + devices = None + for i in range(count): + libusb_get_device_descriptor(dev_array[i], descr) + if idVendor and descr.idVendor != idVendor: + continue + if idProduct and descr.idProduct != idProduct: + continue + device = Device(dev_array[i], descr) + if custom_match and not custom_match(device): + continue + if not find_all: + return device + if not devices: + devices = [] + devices.append(device) + return devices diff --git a/unix-ffi/pyusb/usb/util.py b/unix-ffi/pyusb/usb/util.py new file mode 100644 index 000000000..04e4763e4 --- /dev/null +++ b/unix-ffi/pyusb/usb/util.py @@ -0,0 +1,16 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George + +import usb.control + + +def claim_interface(device, interface): + device._claim_interface(interface) + + +def get_string(device, index): + bs = usb.control.get_descriptor(device, 254, 3, index, 0) + s = "" + for i in range(2, bs[0] & 0xFE, 2): + s += chr(bs[i] | bs[i + 1] << 8) + return s From 1f019f90eaef146960988319166076e12e689e81 Mon Sep 17 00:00:00 2001 From: Mirza Kapetanovic Date: Wed, 12 Jun 2024 11:41:04 +0200 Subject: [PATCH 21/88] requests: Make possible to override headers and allow raw data upload. This removes all the hard-coded request headers from the requests module so they can be overridden by user provided headers dict. Furthermore allow streaming request data without chunk encoding in those cases where content length is known but it's not desirable to load the whole content into memory. Also some servers (e.g. nginx) reject HTTP/1.0 requests with the Transfer-Encoding header set. The change should be backwards compatible as long as the user hasn't provided any of the previously hard-coded headers. Signed-off-by: Mirza Kapetanovic --- python-ecosys/requests/manifest.py | 2 +- python-ecosys/requests/requests/__init__.py | 55 ++++--- python-ecosys/requests/test_requests.py | 155 ++++++++++++++++++++ 3 files changed, 193 insertions(+), 19 deletions(-) create mode 100644 python-ecosys/requests/test_requests.py diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index 97df1560e..eb7bb2d42 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.9.0", pypi="requests") +metadata(version="0.10.0", pypi="requests") package("requests") diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index 740102916..b6bf515dd 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -38,12 +38,15 @@ def request( url, data=None, json=None, - headers={}, + headers=None, stream=None, auth=None, timeout=None, parse_headers=True, ): + if headers is None: + headers = {} + redirect = None # redirection url, None means no redirection chunked_data = data and getattr(data, "__next__", None) and not getattr(data, "__len__", None) @@ -94,33 +97,49 @@ def request( context.verify_mode = tls.CERT_NONE s = context.wrap_socket(s, server_hostname=host) s.write(b"%s /%s HTTP/1.0\r\n" % (method, path)) + if "Host" not in headers: - s.write(b"Host: %s\r\n" % host) - # Iterate over keys to avoid tuple alloc - for k in headers: - s.write(k) - s.write(b": ") - s.write(headers[k]) - s.write(b"\r\n") + headers["Host"] = host + if json is not None: assert data is None import ujson data = ujson.dumps(json) - s.write(b"Content-Type: application/json\r\n") + + if "Content-Type" not in headers: + headers["Content-Type"] = "application/json" + if data: if chunked_data: - s.write(b"Transfer-Encoding: chunked\r\n") - else: - s.write(b"Content-Length: %d\r\n" % len(data)) - s.write(b"Connection: close\r\n\r\n") + if "Transfer-Encoding" not in headers and "Content-Length" not in headers: + headers["Transfer-Encoding"] = "chunked" + elif "Content-Length" not in headers: + headers["Content-Length"] = str(len(data)) + + if "Connection" not in headers: + headers["Connection"] = "close" + + # Iterate over keys to avoid tuple alloc + for k in headers: + s.write(k) + s.write(b": ") + s.write(headers[k]) + s.write(b"\r\n") + + s.write(b"\r\n") + if data: if chunked_data: - for chunk in data: - s.write(b"%x\r\n" % len(chunk)) - s.write(chunk) - s.write(b"\r\n") - s.write("0\r\n\r\n") + if headers.get("Transfer-Encoding", None) == "chunked": + for chunk in data: + s.write(b"%x\r\n" % len(chunk)) + s.write(chunk) + s.write(b"\r\n") + s.write("0\r\n\r\n") + else: + for chunk in data: + s.write(chunk) else: s.write(data) diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py new file mode 100644 index 000000000..540d335cc --- /dev/null +++ b/python-ecosys/requests/test_requests.py @@ -0,0 +1,155 @@ +import io +import sys + + +class Socket: + def __init__(self): + self._write_buffer = io.BytesIO() + self._read_buffer = io.BytesIO(b"HTTP/1.0 200 OK\r\n\r\n") + + def connect(self, address): + pass + + def write(self, buf): + self._write_buffer.write(buf) + + def readline(self): + return self._read_buffer.readline() + + +class usocket: + AF_INET = 2 + SOCK_STREAM = 1 + IPPROTO_TCP = 6 + + @staticmethod + def getaddrinfo(host, port, af=0, type=0, flags=0): + return [(usocket.AF_INET, usocket.SOCK_STREAM, usocket.IPPROTO_TCP, "", ("127.0.0.1", 80))] + + def socket(af=AF_INET, type=SOCK_STREAM, proto=IPPROTO_TCP): + return Socket() + + +sys.modules["usocket"] = usocket +# ruff: noqa: E402 +import requests + + +def format_message(response): + return response.raw._write_buffer.getvalue().decode("utf8") + + +def test_simple_get(): + response = requests.request("GET", "http://example.com/") + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + b"Connection: close\r\n" + b"Host: example.com\r\n\r\n" + ), format_message(response) + + +def test_get_auth(): + response = requests.request( + "GET", "http://example.com/", auth=("test-username", "test-password") + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Host: example.com\r\n" + + b"Authorization: Basic dGVzdC11c2VybmFtZTp0ZXN0LXBhc3N3b3Jk\r\n" + + b"Connection: close\r\n\r\n" + ), format_message(response) + + +def test_get_custom_header(): + response = requests.request("GET", "http://example.com/", headers={"User-Agent": "test-agent"}) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"User-Agent: test-agent\r\n" + + b"Host: example.com\r\n" + + b"Connection: close\r\n\r\n" + ), format_message(response) + + +def test_post_json(): + response = requests.request("GET", "http://example.com/", json="test") + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Connection: close\r\n" + + b"Content-Type: application/json\r\n" + + b"Host: example.com\r\n" + + b"Content-Length: 6\r\n\r\n" + + b'"test"' + ), format_message(response) + + +def test_post_chunked_data(): + def chunks(): + yield "test" + + response = requests.request("GET", "http://example.com/", data=chunks()) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Transfer-Encoding: chunked\r\n" + + b"Host: example.com\r\n" + + b"Connection: close\r\n\r\n" + + b"4\r\ntest\r\n" + + b"0\r\n\r\n" + ), format_message(response) + + +def test_overwrite_get_headers(): + response = requests.request( + "GET", "http://example.com/", headers={"Connection": "keep-alive", "Host": "test.com"} + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + b"Host: test.com\r\n" + b"Connection: keep-alive\r\n\r\n" + ), format_message(response) + + +def test_overwrite_post_json_headers(): + response = requests.request( + "GET", + "http://example.com/", + json="test", + headers={"Content-Type": "text/plain", "Content-Length": "10"}, + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Connection: close\r\n" + + b"Content-Length: 10\r\n" + + b"Content-Type: text/plain\r\n" + + b"Host: example.com\r\n\r\n" + + b'"test"' + ), format_message(response) + + +def test_overwrite_post_chunked_data_headers(): + def chunks(): + yield "test" + + response = requests.request( + "GET", "http://example.com/", data=chunks(), headers={"Content-Length": "4"} + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Host: example.com\r\n" + + b"Content-Length: 4\r\n" + + b"Connection: close\r\n\r\n" + + b"test" + ), format_message(response) + + +test_simple_get() +test_get_auth() +test_get_custom_header() +test_post_json() +test_post_chunked_data() +test_overwrite_get_headers() +test_overwrite_post_json_headers() +test_overwrite_post_chunked_data_headers() From 7271f1ddc75d4d64341e9701a7a21e9b39968b4b Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 12 Jun 2024 13:44:20 +1000 Subject: [PATCH 22/88] all: Change use of "uasyncio" to "asyncio". Signed-off-by: Damien George --- micropython/aioespnow/README.md | 4 ++-- micropython/aioespnow/aioespnow.py | 6 +++--- micropython/aiorepl/aiorepl.py | 2 +- micropython/bluetooth/aioble/aioble/central.py | 2 +- micropython/bluetooth/aioble/aioble/client.py | 2 +- micropython/bluetooth/aioble/aioble/device.py | 2 +- micropython/bluetooth/aioble/aioble/l2cap.py | 2 +- micropython/bluetooth/aioble/aioble/peripheral.py | 2 +- micropython/bluetooth/aioble/aioble/security.py | 2 +- micropython/bluetooth/aioble/aioble/server.py | 2 +- micropython/bluetooth/aioble/examples/l2cap_file_client.py | 2 +- micropython/bluetooth/aioble/examples/l2cap_file_server.py | 2 +- micropython/bluetooth/aioble/examples/temp_client.py | 2 +- micropython/bluetooth/aioble/examples/temp_sensor.py | 2 +- .../aioble/multitests/ble_buffered_characteristic.py | 2 +- .../bluetooth/aioble/multitests/ble_characteristic.py | 2 +- micropython/bluetooth/aioble/multitests/ble_descriptor.py | 2 +- micropython/bluetooth/aioble/multitests/ble_notify.py | 2 +- micropython/bluetooth/aioble/multitests/ble_shutdown.py | 2 +- .../bluetooth/aioble/multitests/ble_write_capture.py | 2 +- micropython/bluetooth/aioble/multitests/ble_write_order.py | 2 +- micropython/bluetooth/aioble/multitests/perf_gatt_notify.py | 2 +- micropython/bluetooth/aioble/multitests/perf_l2cap.py | 2 +- micropython/uaiohttpclient/README | 2 +- micropython/uaiohttpclient/example.py | 2 +- micropython/uaiohttpclient/manifest.py | 2 +- micropython/uaiohttpclient/uaiohttpclient.py | 2 +- 27 files changed, 30 insertions(+), 30 deletions(-) diff --git a/micropython/aioespnow/README.md b/micropython/aioespnow/README.md index a68e765af..132bce103 100644 --- a/micropython/aioespnow/README.md +++ b/micropython/aioespnow/README.md @@ -4,7 +4,7 @@ A supplementary module which extends the micropython `espnow` module to provide `asyncio` support. - Asyncio support is available on all ESP32 targets as well as those ESP8266 -boards which include the `uasyncio` module (ie. ESP8266 devices with at least +boards which include the `asyncio` module (ie. ESP8266 devices with at least 2MB flash storage). ## API reference @@ -52,7 +52,7 @@ A small async server example:: ```python import network import aioespnow - import uasyncio as asyncio + import asyncio # A WLAN interface must be active to send()/recv() network.WLAN(network.STA_IF).active(True) diff --git a/micropython/aioespnow/aioespnow.py b/micropython/aioespnow/aioespnow.py index c00c6fb2b..dec925de2 100644 --- a/micropython/aioespnow/aioespnow.py +++ b/micropython/aioespnow/aioespnow.py @@ -1,12 +1,12 @@ # aioespnow module for MicroPython on ESP32 and ESP8266 # MIT license; Copyright (c) 2022 Glenn Moloney @glenn20 -import uasyncio as asyncio +import asyncio import espnow -# Modelled on the uasyncio.Stream class (extmod/stream/stream.py) -# NOTE: Relies on internal implementation of uasyncio.core (_io_queue) +# Modelled on the asyncio.Stream class (extmod/asyncio/stream.py) +# NOTE: Relies on internal implementation of asyncio.core (_io_queue) class AIOESPNow(espnow.ESPNow): # Read one ESPNow message async def arecv(self): diff --git a/micropython/aiorepl/aiorepl.py b/micropython/aiorepl/aiorepl.py index 14d5d55bc..8f45dfac0 100644 --- a/micropython/aiorepl/aiorepl.py +++ b/micropython/aiorepl/aiorepl.py @@ -41,7 +41,7 @@ async def execute(code, g, s): code = "return {}".format(code) code = """ -import uasyncio as asyncio +import asyncio async def __code(): {} diff --git a/micropython/bluetooth/aioble/aioble/central.py b/micropython/bluetooth/aioble/aioble/central.py index 2f1492d08..6d90cd0f8 100644 --- a/micropython/bluetooth/aioble/aioble/central.py +++ b/micropython/bluetooth/aioble/aioble/central.py @@ -6,7 +6,7 @@ import bluetooth import struct -import uasyncio as asyncio +import asyncio from .core import ( ensure_active, diff --git a/micropython/bluetooth/aioble/aioble/client.py b/micropython/bluetooth/aioble/aioble/client.py index ccde03527..859c6e937 100644 --- a/micropython/bluetooth/aioble/aioble/client.py +++ b/micropython/bluetooth/aioble/aioble/client.py @@ -3,7 +3,7 @@ from micropython import const from collections import deque -import uasyncio as asyncio +import asyncio import struct import bluetooth diff --git a/micropython/bluetooth/aioble/aioble/device.py b/micropython/bluetooth/aioble/aioble/device.py index 8844eb42a..d02d6385f 100644 --- a/micropython/bluetooth/aioble/aioble/device.py +++ b/micropython/bluetooth/aioble/aioble/device.py @@ -3,7 +3,7 @@ from micropython import const -import uasyncio as asyncio +import asyncio import binascii from .core import ble, register_irq_handler, log_error diff --git a/micropython/bluetooth/aioble/aioble/l2cap.py b/micropython/bluetooth/aioble/aioble/l2cap.py index 713c441fd..e2d3bd9d4 100644 --- a/micropython/bluetooth/aioble/aioble/l2cap.py +++ b/micropython/bluetooth/aioble/aioble/l2cap.py @@ -3,7 +3,7 @@ from micropython import const -import uasyncio as asyncio +import asyncio from .core import ble, log_error, register_irq_handler from .device import DeviceConnection diff --git a/micropython/bluetooth/aioble/aioble/peripheral.py b/micropython/bluetooth/aioble/aioble/peripheral.py index a156ccd21..d3dda8bcb 100644 --- a/micropython/bluetooth/aioble/aioble/peripheral.py +++ b/micropython/bluetooth/aioble/aioble/peripheral.py @@ -6,7 +6,7 @@ import bluetooth import struct -import uasyncio as asyncio +import asyncio from .core import ( ensure_active, diff --git a/micropython/bluetooth/aioble/aioble/security.py b/micropython/bluetooth/aioble/aioble/security.py index a0b46e6d6..8e04d5b7b 100644 --- a/micropython/bluetooth/aioble/aioble/security.py +++ b/micropython/bluetooth/aioble/aioble/security.py @@ -2,7 +2,7 @@ # MIT license; Copyright (c) 2021 Jim Mussared from micropython import const, schedule -import uasyncio as asyncio +import asyncio import binascii import json diff --git a/micropython/bluetooth/aioble/aioble/server.py b/micropython/bluetooth/aioble/aioble/server.py index 403700c5a..5d5d7399b 100644 --- a/micropython/bluetooth/aioble/aioble/server.py +++ b/micropython/bluetooth/aioble/aioble/server.py @@ -4,7 +4,7 @@ from micropython import const from collections import deque import bluetooth -import uasyncio as asyncio +import asyncio from .core import ( ensure_active, diff --git a/micropython/bluetooth/aioble/examples/l2cap_file_client.py b/micropython/bluetooth/aioble/examples/l2cap_file_client.py index 2a75bc308..9dce349a7 100644 --- a/micropython/bluetooth/aioble/examples/l2cap_file_client.py +++ b/micropython/bluetooth/aioble/examples/l2cap_file_client.py @@ -10,7 +10,7 @@ from micropython import const -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/examples/l2cap_file_server.py b/micropython/bluetooth/aioble/examples/l2cap_file_server.py index 0c45bd1ff..fb806effc 100644 --- a/micropython/bluetooth/aioble/examples/l2cap_file_server.py +++ b/micropython/bluetooth/aioble/examples/l2cap_file_server.py @@ -21,7 +21,7 @@ from micropython import const -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/examples/temp_client.py b/micropython/bluetooth/aioble/examples/temp_client.py index 42752d8c9..0840359f8 100644 --- a/micropython/bluetooth/aioble/examples/temp_client.py +++ b/micropython/bluetooth/aioble/examples/temp_client.py @@ -5,7 +5,7 @@ from micropython import const -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/examples/temp_sensor.py b/micropython/bluetooth/aioble/examples/temp_sensor.py index 29f774bee..54580f595 100644 --- a/micropython/bluetooth/aioble/examples/temp_sensor.py +++ b/micropython/bluetooth/aioble/examples/temp_sensor.py @@ -5,7 +5,7 @@ from micropython import const -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/multitests/ble_buffered_characteristic.py b/micropython/bluetooth/aioble/multitests/ble_buffered_characteristic.py index 91307908f..e41c3fd1e 100644 --- a/micropython/bluetooth/aioble/multitests/ble_buffered_characteristic.py +++ b/micropython/bluetooth/aioble/multitests/ble_buffered_characteristic.py @@ -9,7 +9,7 @@ import machine import time -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/multitests/ble_characteristic.py b/micropython/bluetooth/aioble/multitests/ble_characteristic.py index b5d1df2fe..0c42bc19b 100644 --- a/micropython/bluetooth/aioble/multitests/ble_characteristic.py +++ b/micropython/bluetooth/aioble/multitests/ble_characteristic.py @@ -9,7 +9,7 @@ import machine import time -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/multitests/ble_descriptor.py b/micropython/bluetooth/aioble/multitests/ble_descriptor.py index 888222ff5..8e32a469a 100644 --- a/micropython/bluetooth/aioble/multitests/ble_descriptor.py +++ b/micropython/bluetooth/aioble/multitests/ble_descriptor.py @@ -9,7 +9,7 @@ import machine import time -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/multitests/ble_notify.py b/micropython/bluetooth/aioble/multitests/ble_notify.py index 200e784c2..6eb85f68c 100644 --- a/micropython/bluetooth/aioble/multitests/ble_notify.py +++ b/micropython/bluetooth/aioble/multitests/ble_notify.py @@ -9,7 +9,7 @@ import machine import time -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/multitests/ble_shutdown.py b/micropython/bluetooth/aioble/multitests/ble_shutdown.py index dea915bfb..28fc53536 100644 --- a/micropython/bluetooth/aioble/multitests/ble_shutdown.py +++ b/micropython/bluetooth/aioble/multitests/ble_shutdown.py @@ -9,7 +9,7 @@ import machine import time -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/multitests/ble_write_capture.py b/micropython/bluetooth/aioble/multitests/ble_write_capture.py index 23c6d4422..0577229e2 100644 --- a/micropython/bluetooth/aioble/multitests/ble_write_capture.py +++ b/micropython/bluetooth/aioble/multitests/ble_write_capture.py @@ -9,7 +9,7 @@ import machine import time -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/multitests/ble_write_order.py b/micropython/bluetooth/aioble/multitests/ble_write_order.py index 10b44bca1..ca47f3837 100644 --- a/micropython/bluetooth/aioble/multitests/ble_write_order.py +++ b/micropython/bluetooth/aioble/multitests/ble_write_order.py @@ -9,7 +9,7 @@ import machine import time -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py b/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py index 3d3159f59..d8a0ea173 100644 --- a/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py +++ b/micropython/bluetooth/aioble/multitests/perf_gatt_notify.py @@ -9,7 +9,7 @@ import machine import time -import uasyncio as asyncio +import asyncio import aioble import bluetooth diff --git a/micropython/bluetooth/aioble/multitests/perf_l2cap.py b/micropython/bluetooth/aioble/multitests/perf_l2cap.py index e21efd6fa..05fd4863e 100644 --- a/micropython/bluetooth/aioble/multitests/perf_l2cap.py +++ b/micropython/bluetooth/aioble/multitests/perf_l2cap.py @@ -7,7 +7,7 @@ import machine import time -import uasyncio as asyncio +import asyncio import aioble import bluetooth import random diff --git a/micropython/uaiohttpclient/README b/micropython/uaiohttpclient/README index a3d88b0a4..1222f9d61 100644 --- a/micropython/uaiohttpclient/README +++ b/micropython/uaiohttpclient/README @@ -1,4 +1,4 @@ -uaiohttpclient is an HTTP client module for MicroPython uasyncio module, +uaiohttpclient is an HTTP client module for MicroPython asyncio module, with API roughly compatible with aiohttp (https://github.com/KeepSafe/aiohttp) module. Note that only client is implemented, for server see picoweb microframework. diff --git a/micropython/uaiohttpclient/example.py b/micropython/uaiohttpclient/example.py index d265c9db7..540d1b3de 100644 --- a/micropython/uaiohttpclient/example.py +++ b/micropython/uaiohttpclient/example.py @@ -2,7 +2,7 @@ # uaiohttpclient - fetch URL passed as command line argument. # import sys -import uasyncio as asyncio +import asyncio import uaiohttpclient as aiohttp diff --git a/micropython/uaiohttpclient/manifest.py b/micropython/uaiohttpclient/manifest.py index a204d57b2..8b35e0a70 100644 --- a/micropython/uaiohttpclient/manifest.py +++ b/micropython/uaiohttpclient/manifest.py @@ -1,4 +1,4 @@ -metadata(description="HTTP client module for MicroPython uasyncio module", version="0.5.2") +metadata(description="HTTP client module for MicroPython asyncio module", version="0.5.2") # Originally written by Paul Sokolovsky. diff --git a/micropython/uaiohttpclient/uaiohttpclient.py b/micropython/uaiohttpclient/uaiohttpclient.py index 6347c3371..2e782638c 100644 --- a/micropython/uaiohttpclient/uaiohttpclient.py +++ b/micropython/uaiohttpclient/uaiohttpclient.py @@ -1,4 +1,4 @@ -import uasyncio as asyncio +import asyncio class ClientResponse: From 84ba4521139157d284015de6b530c13a062caf74 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 12 Jun 2024 13:58:32 +1000 Subject: [PATCH 23/88] all: Use non-u versions of built-in modules. This changes almost all uses of "u-module" to just "module" for the following built-in modules: - binascii - collections - errno - io - json - socket - struct - sys - time There are some remaining uses of "u-module" naming, for the cases where the built-in module is extended in Python, eg `python-stdlib/os` uses `uos`. Also, there are remaining uses of `utime` when non-standard (compared to CPython) functions are used, like `utime.ticks_ms()`. Signed-off-by: Damien George --- .../drivers/display/lcd160cr/lcd160cr.py | 12 ++++++------ .../drivers/radio/nrf24l01/nrf24l01test.py | 14 +++++++------- micropython/net/ntptime/ntptime.py | 17 +++++------------ micropython/udnspkt/example_resolve.py | 14 +++++++------- micropython/udnspkt/udnspkt.py | 3 --- micropython/umqtt.robust/umqtt/robust.py | 4 ++-- micropython/umqtt.simple/example_pub_button.py | 4 ++-- micropython/umqtt.simple/example_sub_led.py | 4 ++-- micropython/umqtt.simple/umqtt/simple.py | 6 +++--- micropython/urllib.urequest/urllib/urequest.py | 6 +++--- python-ecosys/requests/requests/__init__.py | 18 +++++++++--------- python-ecosys/requests/test_requests.py | 6 +++--- python-stdlib/argparse/argparse.py | 2 +- python-stdlib/binascii/test_binascii.py | 6 +++--- python-stdlib/copy/copy.py | 2 +- python-stdlib/pkg_resources/pkg_resources.py | 8 ++++---- unix-ffi/machine/example_timer.py | 4 ++-- unix-ffi/machine/machine/timer.py | 2 -- unix-ffi/os/os/__init__.py | 2 +- unix-ffi/pwd/pwd.py | 8 ++++---- unix-ffi/select/select.py | 2 +- unix-ffi/time/time.py | 12 ++++++------ 22 files changed, 72 insertions(+), 84 deletions(-) diff --git a/micropython/drivers/display/lcd160cr/lcd160cr.py b/micropython/drivers/display/lcd160cr/lcd160cr.py index b86cbff0d..42b5e215b 100644 --- a/micropython/drivers/display/lcd160cr/lcd160cr.py +++ b/micropython/drivers/display/lcd160cr/lcd160cr.py @@ -5,7 +5,7 @@ import machine from utime import sleep_ms from ustruct import calcsize, pack_into -import uerrno +import errno # for set_orient PORTRAIT = const(0) @@ -110,7 +110,7 @@ def _waitfor(self, n, buf): return t -= 1 sleep_ms(1) - raise OSError(uerrno.ETIMEDOUT) + raise OSError(errno.ETIMEDOUT) def oflush(self, n=255): t = 5000 @@ -121,7 +121,7 @@ def oflush(self, n=255): return t -= 1 machine.idle() - raise OSError(uerrno.ETIMEDOUT) + raise OSError(errno.ETIMEDOUT) def iflush(self): t = 5000 @@ -131,7 +131,7 @@ def iflush(self): return t -= 1 sleep_ms(1) - raise OSError(uerrno.ETIMEDOUT) + raise OSError(errno.ETIMEDOUT) #### MISC METHODS #### @@ -254,7 +254,7 @@ def get_pixel(self, x, y): return self.buf[3][1] | self.buf[3][2] << 8 t -= 1 sleep_ms(1) - raise OSError(uerrno.ETIMEDOUT) + raise OSError(errno.ETIMEDOUT) def get_line(self, x, y, buf): l = len(buf) // 2 @@ -268,7 +268,7 @@ def get_line(self, x, y, buf): return t -= 1 sleep_ms(1) - raise OSError(uerrno.ETIMEDOUT) + raise OSError(errno.ETIMEDOUT) def screen_dump(self, buf, x=0, y=0, w=None, h=None): if w is None: diff --git a/micropython/drivers/radio/nrf24l01/nrf24l01test.py b/micropython/drivers/radio/nrf24l01/nrf24l01test.py index ad3e1f67a..a0c4b76f4 100644 --- a/micropython/drivers/radio/nrf24l01/nrf24l01test.py +++ b/micropython/drivers/radio/nrf24l01/nrf24l01test.py @@ -1,7 +1,7 @@ """Test for nrf24l01 module. Portable between MicroPython targets.""" -import usys -import ustruct as struct +import sys +import struct import utime from machine import Pin, SPI, SoftSPI from nrf24l01 import NRF24L01 @@ -14,20 +14,20 @@ # initiator may be a slow device. Value tested with Pyboard, ESP32 and ESP8266. _RESPONDER_SEND_DELAY = const(10) -if usys.platform == "pyboard": +if sys.platform == "pyboard": spi = SPI(2) # miso : Y7, mosi : Y8, sck : Y6 cfg = {"spi": spi, "csn": "Y5", "ce": "Y4"} -elif usys.platform == "esp8266": # Hardware SPI +elif sys.platform == "esp8266": # Hardware SPI spi = SPI(1) # miso : 12, mosi : 13, sck : 14 cfg = {"spi": spi, "csn": 4, "ce": 5} -elif usys.platform == "esp32": # Software SPI +elif sys.platform == "esp32": # Software SPI spi = SoftSPI(sck=Pin(25), mosi=Pin(33), miso=Pin(32)) cfg = {"spi": spi, "csn": 26, "ce": 27} -elif usys.platform == "rp2": # Hardware SPI with explicit pin definitions +elif sys.platform == "rp2": # Hardware SPI with explicit pin definitions spi = SPI(0, sck=Pin(2), mosi=Pin(3), miso=Pin(4)) cfg = {"spi": spi, "csn": 5, "ce": 6} else: - raise ValueError("Unsupported platform {}".format(usys.platform)) + raise ValueError("Unsupported platform {}".format(sys.platform)) # Addresses are in little-endian format. They correspond to big-endian # 0xf0f0f0f0e1, 0xf0f0f0f0d2 diff --git a/micropython/net/ntptime/ntptime.py b/micropython/net/ntptime/ntptime.py index 25cc62ad1..d77214d1d 100644 --- a/micropython/net/ntptime/ntptime.py +++ b/micropython/net/ntptime/ntptime.py @@ -1,13 +1,6 @@ -import utime - -try: - import usocket as socket -except: - import socket -try: - import ustruct as struct -except: - import struct +from time import gmtime +import socket +import struct # The NTP host can be configured at runtime by doing: ntptime.host = 'myhost.org' host = "pool.ntp.org" @@ -53,7 +46,7 @@ def time(): # Convert timestamp from NTP format to our internal format - EPOCH_YEAR = utime.gmtime(0)[0] + EPOCH_YEAR = gmtime(0)[0] if EPOCH_YEAR == 2000: # (date(2000, 1, 1) - date(1900, 1, 1)).days * 24*60*60 NTP_DELTA = 3155673600 @@ -71,5 +64,5 @@ def settime(): t = time() import machine - tm = utime.gmtime(t) + tm = gmtime(t) machine.RTC().datetime((tm[0], tm[1], tm[2], tm[6] + 1, tm[3], tm[4], tm[5], 0)) diff --git a/micropython/udnspkt/example_resolve.py b/micropython/udnspkt/example_resolve.py index c1215045a..d72c17a48 100644 --- a/micropython/udnspkt/example_resolve.py +++ b/micropython/udnspkt/example_resolve.py @@ -1,15 +1,15 @@ -import uio -import usocket +import io +import socket import udnspkt -s = usocket.socket(usocket.AF_INET, usocket.SOCK_DGRAM) -dns_addr = usocket.getaddrinfo("127.0.0.1", 53)[0][-1] +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +dns_addr = socket.getaddrinfo("127.0.0.1", 53)[0][-1] def resolve(domain, is_ipv6): - buf = uio.BytesIO(48) + buf = io.BytesIO(48) udnspkt.make_req(buf, "google.com", is_ipv6) v = buf.getvalue() print("query: ", v) @@ -17,11 +17,11 @@ def resolve(domain, is_ipv6): resp = s.recv(1024) print("resp:", resp) - buf = uio.BytesIO(resp) + buf = io.BytesIO(resp) addr = udnspkt.parse_resp(buf, is_ipv6) print("bin addr:", addr) - print("addr:", usocket.inet_ntop(usocket.AF_INET6 if is_ipv6 else usocket.AF_INET, addr)) + print("addr:", socket.inet_ntop(socket.AF_INET6 if is_ipv6 else socket.AF_INET, addr)) resolve("google.com", False) diff --git a/micropython/udnspkt/udnspkt.py b/micropython/udnspkt/udnspkt.py index 2cb11ab92..f3b998a8a 100644 --- a/micropython/udnspkt/udnspkt.py +++ b/micropython/udnspkt/udnspkt.py @@ -1,6 +1,3 @@ -import uio - - def write_fqdn(buf, name): parts = name.split(".") for p in parts: diff --git a/micropython/umqtt.robust/umqtt/robust.py b/micropython/umqtt.robust/umqtt/robust.py index 4cc10e336..51596de9e 100644 --- a/micropython/umqtt.robust/umqtt/robust.py +++ b/micropython/umqtt.robust/umqtt/robust.py @@ -1,4 +1,4 @@ -import utime +import time from . import simple @@ -7,7 +7,7 @@ class MQTTClient(simple.MQTTClient): DEBUG = False def delay(self, i): - utime.sleep(self.DELAY) + time.sleep(self.DELAY) def log(self, in_reconnect, e): if self.DEBUG: diff --git a/micropython/umqtt.simple/example_pub_button.py b/micropython/umqtt.simple/example_pub_button.py index 1bc47bc5e..2a3ec851e 100644 --- a/micropython/umqtt.simple/example_pub_button.py +++ b/micropython/umqtt.simple/example_pub_button.py @@ -1,5 +1,5 @@ import time -import ubinascii +import binascii import machine from umqtt.simple import MQTTClient from machine import Pin @@ -10,7 +10,7 @@ # Default MQTT server to connect to SERVER = "192.168.1.35" -CLIENT_ID = ubinascii.hexlify(machine.unique_id()) +CLIENT_ID = binascii.hexlify(machine.unique_id()) TOPIC = b"led" diff --git a/micropython/umqtt.simple/example_sub_led.py b/micropython/umqtt.simple/example_sub_led.py index 73c6b58d8..c3dcf08d2 100644 --- a/micropython/umqtt.simple/example_sub_led.py +++ b/micropython/umqtt.simple/example_sub_led.py @@ -1,6 +1,6 @@ from umqtt.simple import MQTTClient from machine import Pin -import ubinascii +import binascii import machine import micropython @@ -11,7 +11,7 @@ # Default MQTT server to connect to SERVER = "192.168.1.35" -CLIENT_ID = ubinascii.hexlify(machine.unique_id()) +CLIENT_ID = binascii.hexlify(machine.unique_id()) TOPIC = b"led" diff --git a/micropython/umqtt.simple/umqtt/simple.py b/micropython/umqtt.simple/umqtt/simple.py index e84e585c4..6da38e445 100644 --- a/micropython/umqtt.simple/umqtt/simple.py +++ b/micropython/umqtt.simple/umqtt/simple.py @@ -1,6 +1,6 @@ -import usocket as socket -import ustruct as struct -from ubinascii import hexlify +import socket +import struct +from binascii import hexlify class MQTTException(Exception): diff --git a/micropython/urllib.urequest/urllib/urequest.py b/micropython/urllib.urequest/urllib/urequest.py index 5154c0f05..f83cbaa94 100644 --- a/micropython/urllib.urequest/urllib/urequest.py +++ b/micropython/urllib.urequest/urllib/urequest.py @@ -1,4 +1,4 @@ -import usocket +import socket def urlopen(url, data=None, method="GET"): @@ -22,10 +22,10 @@ def urlopen(url, data=None, method="GET"): host, port = host.split(":", 1) port = int(port) - ai = usocket.getaddrinfo(host, port, 0, usocket.SOCK_STREAM) + ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) ai = ai[0] - s = usocket.socket(ai[0], ai[1], ai[2]) + s = socket.socket(ai[0], ai[1], ai[2]) try: s.connect(ai[-1]) if proto == "https:": diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index b6bf515dd..a9a183619 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -1,4 +1,4 @@ -import usocket +import socket class Response: @@ -28,9 +28,9 @@ def text(self): return str(self.content, self.encoding) def json(self): - import ujson + import json - return ujson.loads(self.content) + return json.loads(self.content) def request( @@ -51,11 +51,11 @@ def request( chunked_data = data and getattr(data, "__next__", None) and not getattr(data, "__len__", None) if auth is not None: - import ubinascii + import binascii username, password = auth formated = b"{}:{}".format(username, password) - formated = str(ubinascii.b2a_base64(formated)[:-1], "ascii") + formated = str(binascii.b2a_base64(formated)[:-1], "ascii") headers["Authorization"] = "Basic {}".format(formated) try: @@ -76,14 +76,14 @@ def request( host, port = host.split(":", 1) port = int(port) - ai = usocket.getaddrinfo(host, port, 0, usocket.SOCK_STREAM) + ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) ai = ai[0] resp_d = None if parse_headers is not False: resp_d = {} - s = usocket.socket(ai[0], usocket.SOCK_STREAM, ai[2]) + s = socket.socket(ai[0], socket.SOCK_STREAM, ai[2]) if timeout is not None: # Note: settimeout is not supported on all platforms, will raise @@ -103,9 +103,9 @@ def request( if json is not None: assert data is None - import ujson + from json import dumps - data = ujson.dumps(json) + data = dumps(json) if "Content-Type" not in headers: headers["Content-Type"] = "application/json" diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py index 540d335cc..513e533a3 100644 --- a/python-ecosys/requests/test_requests.py +++ b/python-ecosys/requests/test_requests.py @@ -17,20 +17,20 @@ def readline(self): return self._read_buffer.readline() -class usocket: +class socket: AF_INET = 2 SOCK_STREAM = 1 IPPROTO_TCP = 6 @staticmethod def getaddrinfo(host, port, af=0, type=0, flags=0): - return [(usocket.AF_INET, usocket.SOCK_STREAM, usocket.IPPROTO_TCP, "", ("127.0.0.1", 80))] + return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", 80))] def socket(af=AF_INET, type=SOCK_STREAM, proto=IPPROTO_TCP): return Socket() -sys.modules["usocket"] = usocket +sys.modules["socket"] = socket # ruff: noqa: E402 import requests diff --git a/python-stdlib/argparse/argparse.py b/python-stdlib/argparse/argparse.py index cb575dd24..5c92887f9 100644 --- a/python-stdlib/argparse/argparse.py +++ b/python-stdlib/argparse/argparse.py @@ -3,7 +3,7 @@ """ import sys -from ucollections import namedtuple +from collections import namedtuple class _ArgError(BaseException): diff --git a/python-stdlib/binascii/test_binascii.py b/python-stdlib/binascii/test_binascii.py index 942ddc51b..075b2ff3c 100644 --- a/python-stdlib/binascii/test_binascii.py +++ b/python-stdlib/binascii/test_binascii.py @@ -1,5 +1,5 @@ from binascii import * -import utime +import time data = b"zlutoucky kun upel dabelske ody" h = hexlify(data) @@ -14,10 +14,10 @@ a2b_base64(b"as==") == b"j" -start = utime.time() +start = time.time() for x in range(100000): d = unhexlify(h) -print("100000 iterations in: " + str(utime.time() - start)) +print("100000 iterations in: " + str(time.time() - start)) print("OK") diff --git a/python-stdlib/copy/copy.py b/python-stdlib/copy/copy.py index f7bfdd6a1..0a9283777 100644 --- a/python-stdlib/copy/copy.py +++ b/python-stdlib/copy/copy.py @@ -62,7 +62,7 @@ class Error(Exception): error = Error # backward compatibility try: - from ucollections import OrderedDict + from collections import OrderedDict except ImportError: OrderedDict = None diff --git a/python-stdlib/pkg_resources/pkg_resources.py b/python-stdlib/pkg_resources/pkg_resources.py index cd3e0fe96..d69cb0577 100644 --- a/python-stdlib/pkg_resources/pkg_resources.py +++ b/python-stdlib/pkg_resources/pkg_resources.py @@ -1,4 +1,4 @@ -import uio +import io c = {} @@ -18,11 +18,11 @@ def resource_stream(package, resource): else: d = "." # if d[0] != "/": - # import uos - # d = uos.getcwd() + "/" + d + # import os + # d = os.getcwd() + "/" + d c[package] = d + "/" p = c[package] if isinstance(p, dict): - return uio.BytesIO(p[resource]) + return io.BytesIO(p[resource]) return open(p + resource, "rb") diff --git a/unix-ffi/machine/example_timer.py b/unix-ffi/machine/example_timer.py index a0d44110f..550d68cd3 100644 --- a/unix-ffi/machine/example_timer.py +++ b/unix-ffi/machine/example_timer.py @@ -1,4 +1,4 @@ -import utime +import time from machine import Timer @@ -7,5 +7,5 @@ t1.callback(lambda t: print(t, "tick1")) t2.callback(lambda t: print(t, "tick2")) -utime.sleep(3) +time.sleep(3) print("done") diff --git a/unix-ffi/machine/machine/timer.py b/unix-ffi/machine/machine/timer.py index 1aa53f936..3f371142c 100644 --- a/unix-ffi/machine/machine/timer.py +++ b/unix-ffi/machine/machine/timer.py @@ -1,9 +1,7 @@ import ffilib import uctypes import array -import uos import os -import utime from signal import * libc = ffilib.libc() diff --git a/unix-ffi/os/os/__init__.py b/unix-ffi/os/os/__init__.py index 3cca078f9..6c87da892 100644 --- a/unix-ffi/os/os/__init__.py +++ b/unix-ffi/os/os/__init__.py @@ -1,5 +1,5 @@ import array -import ustruct as struct +import struct import errno as errno_ import stat as stat_ import ffilib diff --git a/unix-ffi/pwd/pwd.py b/unix-ffi/pwd/pwd.py index 29ebe3416..561269ed2 100644 --- a/unix-ffi/pwd/pwd.py +++ b/unix-ffi/pwd/pwd.py @@ -1,8 +1,8 @@ import ffilib import uctypes -import ustruct +import struct -from ucollections import namedtuple +from collections import namedtuple libc = ffilib.libc() @@ -20,6 +20,6 @@ def getpwnam(user): if not passwd: raise KeyError("getpwnam(): name not found: {}".format(user)) passwd_fmt = "SSIISSS" - passwd = uctypes.bytes_at(passwd, ustruct.calcsize(passwd_fmt)) - passwd = ustruct.unpack(passwd_fmt, passwd) + passwd = uctypes.bytes_at(passwd, struct.calcsize(passwd_fmt)) + passwd = struct.unpack(passwd_fmt, passwd) return struct_passwd(*passwd) diff --git a/unix-ffi/select/select.py b/unix-ffi/select/select.py index eec9bfb81..9d514a31d 100644 --- a/unix-ffi/select/select.py +++ b/unix-ffi/select/select.py @@ -1,5 +1,5 @@ import ffi -import ustruct as struct +import struct import os import errno import ffilib diff --git a/unix-ffi/time/time.py b/unix-ffi/time/time.py index 075d904f5..319228dc8 100644 --- a/unix-ffi/time/time.py +++ b/unix-ffi/time/time.py @@ -1,6 +1,6 @@ from utime import * -from ucollections import namedtuple -import ustruct +from collections import namedtuple +import struct import uctypes import ffi import ffilib @@ -34,13 +34,13 @@ def _tuple_to_c_tm(t): - return ustruct.pack( + return struct.pack( "@iiiiiiiii", t[5], t[4], t[3], t[2], t[1] - 1, t[0] - 1900, (t[6] + 1) % 7, t[7] - 1, t[8] ) def _c_tm_to_tuple(tm): - t = ustruct.unpack("@iiiiiiiii", tm) + t = struct.unpack("@iiiiiiiii", tm) return _struct_time( t[5] + 1900, t[4] + 1, t[3], t[2], t[1], t[0], (t[6] - 1) % 7, t[7] + 1, t[8] ) @@ -64,7 +64,7 @@ def localtime(t=None): t = time() t = int(t) - a = ustruct.pack("l", t) + a = struct.pack("l", t) tm_p = localtime_(a) return _c_tm_to_tuple(uctypes.bytearray_at(tm_p, 36)) @@ -74,7 +74,7 @@ def gmtime(t=None): t = time() t = int(t) - a = ustruct.pack("l", t) + a = struct.pack("l", t) tm_p = gmtime_(a) return _c_tm_to_tuple(uctypes.bytearray_at(tm_p, 36)) From 2b3bd5b7e0da893ed9ef3c10951800960cb972ae Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 12 Jun 2024 14:44:48 +1000 Subject: [PATCH 24/88] aioble/multitests: Store a reference to tasks and cancel when done. Storing references to tasks is required by CPython, and enforced by Ruff RUF006. In this case it's also reasonable to cancel these tasks once the test is finished. Signed-off-by: Damien George --- .../bluetooth/aioble/multitests/ble_write_order.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/micropython/bluetooth/aioble/multitests/ble_write_order.py b/micropython/bluetooth/aioble/multitests/ble_write_order.py index ca47f3837..24da54c2d 100644 --- a/micropython/bluetooth/aioble/multitests/ble_write_order.py +++ b/micropython/bluetooth/aioble/multitests/ble_write_order.py @@ -44,12 +44,12 @@ async def instance0_task(): # Register characteristic.written() handlers as asyncio background tasks. # The order of these is important! - asyncio.create_task(task_written(characteristic_second, "second")) - asyncio.create_task(task_written(characteristic_first, "first")) + task_second = asyncio.create_task(task_written(characteristic_second, "second")) + task_first = asyncio.create_task(task_written(characteristic_first, "first")) # This dummy task simulates background processing on a real system that # can block the asyncio loop for brief periods of time - asyncio.create_task(task_dummy()) + task_dummy_ = asyncio.create_task(task_dummy()) multitest.globals(BDADDR=aioble.config("mac")) multitest.next() @@ -63,6 +63,10 @@ async def instance0_task(): await connection.disconnected() + task_second.cancel() + task_first.cancel() + task_dummy_.cancel() + async def task_written(chr, label): while True: From 0b0e0cc2df253e42abffaf04ef5dc5d347f53101 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 16 Jun 2024 10:30:48 +1000 Subject: [PATCH 25/88] quopri: Remove dependency on test.support and subprocess in unit test. Signed-off-by: Damien George --- python-stdlib/quopri/test_quopri.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/python-stdlib/quopri/test_quopri.py b/python-stdlib/quopri/test_quopri.py index 5655dd8b0..b87e54842 100644 --- a/python-stdlib/quopri/test_quopri.py +++ b/python-stdlib/quopri/test_quopri.py @@ -1,7 +1,6 @@ -from test import support import unittest -import sys, os, io, subprocess +import sys, os, io import quopri @@ -193,7 +192,8 @@ def test_decode_header(self): for p, e in self.HSTRINGS: self.assertEqual(quopri.decodestring(e, header=True), p) - def _test_scriptencode(self): + @unittest.skip("requires subprocess") + def test_scriptencode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen( [sys.executable, "-mquopri"], stdin=subprocess.PIPE, stdout=subprocess.PIPE @@ -210,7 +210,8 @@ def _test_scriptencode(self): self.assertEqual(cout[i], e[i]) self.assertEqual(cout, e) - def _test_scriptdecode(self): + @unittest.skip("requires subprocess") + def test_scriptdecode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen( [sys.executable, "-mquopri", "-d"], stdin=subprocess.PIPE, stdout=subprocess.PIPE @@ -220,11 +221,3 @@ def _test_scriptdecode(self): cout = cout.decode("latin-1") p = p.decode("latin-1") self.assertEqual(cout.splitlines(), p.splitlines()) - - -def test_main(): - support.run_unittest(QuopriTestCase) - - -if __name__ == "__main__": - test_main() From 469b81b567b9bb81ff6236c40f2959411e9ef1e5 Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 16 Jun 2024 10:41:05 +1000 Subject: [PATCH 26/88] contextlib: Use a list instead of deque for exit callbacks. Since deque was removed from this repository the built-in one needs to be used, and that doesn't have unbounded growth. So use a list instead, which is adequate becasue contextlib only needs append and pop, not double ended behaviour (the previous pure-Python implementation of deque that was used here anyway used a list as its storage container). Also tweak the excessive-nesting test so it uses less memory and can run on the unix port. Signed-off-by: Damien George --- python-stdlib/contextlib/contextlib.py | 4 ++-- python-stdlib/contextlib/manifest.py | 2 +- python-stdlib/contextlib/tests.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/python-stdlib/contextlib/contextlib.py b/python-stdlib/contextlib/contextlib.py index 2b2020357..3e598b4b6 100644 --- a/python-stdlib/contextlib/contextlib.py +++ b/python-stdlib/contextlib/contextlib.py @@ -85,13 +85,13 @@ class ExitStack(object): """ def __init__(self): - self._exit_callbacks = deque() + self._exit_callbacks = [] def pop_all(self): """Preserve the context stack by transferring it to a new instance""" new_stack = type(self)() new_stack._exit_callbacks = self._exit_callbacks - self._exit_callbacks = deque() + self._exit_callbacks = [] return new_stack def _push_cm_exit(self, cm, cm_exit): diff --git a/python-stdlib/contextlib/manifest.py b/python-stdlib/contextlib/manifest.py index 2894ec5c4..3e05bca18 100644 --- a/python-stdlib/contextlib/manifest.py +++ b/python-stdlib/contextlib/manifest.py @@ -1,4 +1,4 @@ -metadata(description="Port of contextlib for micropython", version="3.4.3") +metadata(description="Port of contextlib for micropython", version="3.4.4") require("ucontextlib") require("collections") diff --git a/python-stdlib/contextlib/tests.py b/python-stdlib/contextlib/tests.py index 19f07add8..c122c452e 100644 --- a/python-stdlib/contextlib/tests.py +++ b/python-stdlib/contextlib/tests.py @@ -399,7 +399,7 @@ def test_exit_exception_chaining_suppress(self): def test_excessive_nesting(self): # The original implementation would die with RecursionError here with ExitStack() as stack: - for i in range(10000): + for i in range(5000): stack.callback(int) def test_instance_bypass(self): From 0d4b3635b4e87bb3a8f36e2e4c0c9198de9963ac Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 16 Jun 2024 10:42:18 +1000 Subject: [PATCH 27/88] datetime: Skip tests that require the host to be in UTC timezone. Signed-off-by: Damien George --- python-stdlib/datetime/test_datetime.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python-stdlib/datetime/test_datetime.py b/python-stdlib/datetime/test_datetime.py index 372bdf3de..98da458f9 100644 --- a/python-stdlib/datetime/test_datetime.py +++ b/python-stdlib/datetime/test_datetime.py @@ -2082,9 +2082,11 @@ def test_timetuple00(self): with LocalTz("Europe/Rome"): self.assertEqual(dt1.timetuple()[:8], (2002, 1, 31, 0, 0, 0, 3, 31)) + @unittest.skip("broken when running with non-UTC timezone") def test_timetuple01(self): self.assertEqual(dt27tz2.timetuple()[:8], (2010, 3, 27, 12, 0, 0, 5, 86)) + @unittest.skip("broken when running with non-UTC timezone") def test_timetuple02(self): self.assertEqual(dt28tz2.timetuple()[:8], (2010, 3, 28, 12, 0, 0, 6, 87)) From f1c7f2885d1da1e0ed07407ff184fd9a9293465a Mon Sep 17 00:00:00 2001 From: Damien George Date: Sun, 16 Jun 2024 10:42:30 +1000 Subject: [PATCH 28/88] fnmatch: Don't require test.support, which no longer exists. Signed-off-by: Damien George --- python-stdlib/fnmatch/test_fnmatch.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/python-stdlib/fnmatch/test_fnmatch.py b/python-stdlib/fnmatch/test_fnmatch.py index 4eaeec63b..97ef8fff7 100644 --- a/python-stdlib/fnmatch/test_fnmatch.py +++ b/python-stdlib/fnmatch/test_fnmatch.py @@ -1,6 +1,5 @@ """Test cases for the fnmatch module.""" -from test import support import unittest from fnmatch import fnmatch, fnmatchcase, translate, filter @@ -79,11 +78,3 @@ def test_translate(self): class FilterTestCase(unittest.TestCase): def test_filter(self): self.assertEqual(filter(["a", "b"], "a"), ["a"]) - - -def main(): - support.run_unittest(FnmatchTestCase, TranslateTestCase, FilterTestCase) - - -if __name__ == "__main__": - main() From 8834023d05c3fb9467875bda372af3afae2d98e9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 17 Jun 2024 11:16:53 +1000 Subject: [PATCH 29/88] hashlib: Only import pure Python hashlib when running test. Signed-off-by: Damien George --- python-stdlib/hashlib/tests/test_sha256.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python-stdlib/hashlib/tests/test_sha256.py b/python-stdlib/hashlib/tests/test_sha256.py index 024821b06..a311a8cc9 100644 --- a/python-stdlib/hashlib/tests/test_sha256.py +++ b/python-stdlib/hashlib/tests/test_sha256.py @@ -1,3 +1,7 @@ +# Prevent importing any built-in hashes, so this test tests only the pure Python hashes. +import sys +sys.modules['uhashlib'] = sys + import unittest from hashlib import sha256 From 98f8a7e77181d9558c0080e6a6c9cf920503e4a3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 14 Jun 2024 11:40:29 +1000 Subject: [PATCH 30/88] github/workflows: Add workflow to run package tests. All of the runable package tests are run together in the new `tools/ci.sh` function called `ci_package_tests_run`. This is added to a new GitHub workflow to test the packages as part of CI. Some packages use `unittest` while others use an ad-hoc test script. Eventually it would be good to unify all the package tests to use `unittest`. Signed-off-by: Damien George --- .github/workflows/package_tests.yml | 16 ++++++ tools/ci.sh | 88 +++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 .github/workflows/package_tests.yml diff --git a/.github/workflows/package_tests.yml b/.github/workflows/package_tests.yml new file mode 100644 index 000000000..5e503509e --- /dev/null +++ b/.github/workflows/package_tests.yml @@ -0,0 +1,16 @@ +name: Package tests + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - name: Setup environment + run: source tools/ci.sh && ci_package_tests_setup_micropython + - name: Setup libraries + run: source tools/ci.sh && ci_package_tests_setup_lib + - name: Run tests + run: source tools/ci.sh && ci_package_tests_run diff --git a/tools/ci.sh b/tools/ci.sh index 761491c6e..c2cf9dbad 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -1,5 +1,7 @@ #!/bin/bash +CP=/bin/cp + ######################################################################################## # commit formatting @@ -12,6 +14,92 @@ function ci_commit_formatting_run { tools/verifygitlog.py -v upstream/master..HEAD --no-merges } +######################################################################################## +# package tests + +MICROPYTHON=/tmp/micropython/ports/unix/build-standard/micropython + +function ci_package_tests_setup_micropython { + git clone https://github.com/micropython/micropython.git /tmp/micropython + + # build mpy-cross and micropython (use -O0 to speed up the build) + make -C /tmp/micropython/mpy-cross -j CFLAGS_EXTRA=-O0 + make -C /tmp/micropython/ports/unix submodules + make -C /tmp/micropython/ports/unix -j CFLAGS_EXTRA=-O0 +} + +function ci_package_tests_setup_lib { + mkdir -p ~/.micropython/lib + $CP micropython/ucontextlib/ucontextlib.py ~/.micropython/lib/ + $CP python-stdlib/fnmatch/fnmatch.py ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-core/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha224/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha256/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha384/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha512/hashlib ~/.micropython/lib/ + $CP python-stdlib/shutil/shutil.py ~/.micropython/lib/ + $CP python-stdlib/tempfile/tempfile.py ~/.micropython/lib/ + $CP -r python-stdlib/unittest/unittest ~/.micropython/lib/ + $CP -r python-stdlib/unittest-discover/unittest ~/.micropython/lib/ + $CP unix-ffi/ffilib/ffilib.py ~/.micropython/lib/ + tree ~/.micropython +} + +function ci_package_tests_run { + for test in \ + micropython/drivers/storage/sdcard/sdtest.py \ + micropython/xmltok/test_xmltok.py \ + python-ecosys/requests/test_requests.py \ + python-stdlib/argparse/test_argparse.py \ + python-stdlib/base64/test_base64.py \ + python-stdlib/binascii/test_binascii.py \ + python-stdlib/collections-defaultdict/test_defaultdict.py \ + python-stdlib/functools/test_partial.py \ + python-stdlib/functools/test_reduce.py \ + python-stdlib/heapq/test_heapq.py \ + python-stdlib/hmac/test_hmac.py \ + python-stdlib/itertools/test_itertools.py \ + python-stdlib/operator/test_operator.py \ + python-stdlib/os-path/test_path.py \ + python-stdlib/pickle/test_pickle.py \ + python-stdlib/string/test_translate.py \ + unix-ffi/gettext/test_gettext.py \ + unix-ffi/pwd/test_getpwnam.py \ + unix-ffi/re/test_re.py \ + unix-ffi/time/test_strftime.py \ + ; do + echo "Running test $test" + (cd `dirname $test` && $MICROPYTHON `basename $test`) + if [ $? -ne 0 ]; then + false # make this function return an error code + return + fi + done + + for path in \ + micropython/ucontextlib \ + python-stdlib/contextlib \ + python-stdlib/datetime \ + python-stdlib/fnmatch \ + python-stdlib/hashlib \ + python-stdlib/pathlib \ + python-stdlib/quopri \ + python-stdlib/shutil \ + python-stdlib/tempfile \ + python-stdlib/time \ + python-stdlib/unittest-discover/tests \ + ; do + (cd $path && $MICROPYTHON -m unittest) + if [ $? -ne 0 ]; then false; return; fi + done + + (cd micropython/usb/usb-device && $MICROPYTHON -m tests.test_core_buffer) + if [ $? -ne 0 ]; then false; return; fi + + (cd python-ecosys/cbor2 && $MICROPYTHON -m examples.cbor_test) + if [ $? -ne 0 ]; then false; return; fi +} + ######################################################################################## # build packages From 50ac49c42b9853ac1f344f092684ea69109f9aff Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Thu, 6 Jun 2024 15:12:45 +1000 Subject: [PATCH 31/88] unittest-discover: Avoid adding test parent dir to sys.path. When running tests from subfolders, import by "full dotted path" rather than just module name, removing the need to add the test parent folder to `sys.path`. This matches CPython more closely, which places `abspath(top)` at the start of `sys.path` but doesn't include the test file parent dir at all. It fixes issues where projects may include a `test_xxx.py` file in their distribution which would (prior to this change) be unintentionally found by unittest-discover. Signed-off-by: Andrew Leech --- python-stdlib/unittest-discover/manifest.py | 2 +- .../unittest-discover/tests/sub/sub.py | 1 + .../tests/sub/test_module_import.py | 13 +++++++++++++ .../unittest-discover/unittest/__main__.py | 19 +++++++++++++------ 4 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 python-stdlib/unittest-discover/tests/sub/sub.py create mode 100644 python-stdlib/unittest-discover/tests/sub/test_module_import.py diff --git a/python-stdlib/unittest-discover/manifest.py b/python-stdlib/unittest-discover/manifest.py index 14bec5201..5610f41e2 100644 --- a/python-stdlib/unittest-discover/manifest.py +++ b/python-stdlib/unittest-discover/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.1.2") +metadata(version="0.1.3") require("argparse") require("fnmatch") diff --git a/python-stdlib/unittest-discover/tests/sub/sub.py b/python-stdlib/unittest-discover/tests/sub/sub.py new file mode 100644 index 000000000..b6614dd63 --- /dev/null +++ b/python-stdlib/unittest-discover/tests/sub/sub.py @@ -0,0 +1 @@ +imported = True diff --git a/python-stdlib/unittest-discover/tests/sub/test_module_import.py b/python-stdlib/unittest-discover/tests/sub/test_module_import.py new file mode 100644 index 000000000..5c6404d6f --- /dev/null +++ b/python-stdlib/unittest-discover/tests/sub/test_module_import.py @@ -0,0 +1,13 @@ +import sys +import unittest + + +class TestModuleImport(unittest.TestCase): + def test_ModuleImportPath(self): + try: + from sub.sub import imported + assert imported + except ImportError: + print("This test is intended to be run with unittest discover" + "from the unittest-discover/tests dir. sys.path:", sys.path) + raise diff --git a/python-stdlib/unittest-discover/unittest/__main__.py b/python-stdlib/unittest-discover/unittest/__main__.py index 8eb173a22..09dfd03b9 100644 --- a/python-stdlib/unittest-discover/unittest/__main__.py +++ b/python-stdlib/unittest-discover/unittest/__main__.py @@ -6,7 +6,12 @@ from fnmatch import fnmatch from micropython import const -from unittest import TestRunner, TestResult, TestSuite +try: + from unittest import TestRunner, TestResult, TestSuite +except ImportError: + print("Error: This must be used from an installed copy of unittest-discover which will" + " also install base unittest module.") + raise # Run a single test in a clean environment. @@ -14,11 +19,11 @@ def _run_test_module(runner: TestRunner, module_name: str, *extra_paths: list[st module_snapshot = {k: v for k, v in sys.modules.items()} path_snapshot = sys.path[:] try: - for path in reversed(extra_paths): + for path in extra_paths: if path: sys.path.insert(0, path) - module = __import__(module_name) + module = __import__(module_name, None, None, module_name) suite = TestSuite(module_name) suite._load_module(module) return runner.run(suite) @@ -36,16 +41,18 @@ def _run_all_in_dir(runner: TestRunner, path: str, pattern: str, top: str): for fname, ftype, *_ in os.ilistdir(path): if fname in ("..", "."): continue + fpath = "/".join((path, fname)) if ftype == _DIR_TYPE: result += _run_all_in_dir( runner=runner, - path="/".join((path, fname)), + path=fpath, pattern=pattern, top=top, ) if fnmatch(fname, pattern): - module_name = fname.rsplit(".", 1)[0] - result += _run_test_module(runner, module_name, path, top) + module_path = fpath.rsplit(".", 1)[0] # remove ext + module_path = module_path.replace("/", ".").strip(".") + result += _run_test_module(runner, module_path, top) return result From b5aa5f0d1bcfcd2f8ab14def0d86204ef02ae705 Mon Sep 17 00:00:00 2001 From: Jared Hancock Date: Mon, 24 Jun 2024 16:51:26 -0500 Subject: [PATCH 32/88] logging: Fix StreamHandler to call parent constructor. Otherwise there's a crash on line 70 where level is not a property of the class unless explicitly set with `setLevel()`. --- python-stdlib/logging/logging.py | 1 + python-stdlib/logging/manifest.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python-stdlib/logging/logging.py b/python-stdlib/logging/logging.py index d17e42c4f..f4874df7d 100644 --- a/python-stdlib/logging/logging.py +++ b/python-stdlib/logging/logging.py @@ -58,6 +58,7 @@ def format(self, record): class StreamHandler(Handler): def __init__(self, stream=None): + super().__init__() self.stream = _stream if stream is None else stream self.terminator = "\n" diff --git a/python-stdlib/logging/manifest.py b/python-stdlib/logging/manifest.py index daf5d9c94..d9f0ee886 100644 --- a/python-stdlib/logging/manifest.py +++ b/python-stdlib/logging/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.6.0") +metadata(version="0.6.1") module("logging.py") From 0a91a37563f6783d3913577705218fcea479edc7 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 18 Jun 2024 17:17:46 +1000 Subject: [PATCH 33/88] usb-device-cdc: Fix lost data in read() path if short reads happened. If the CDC receive buffer was full and some code read less than 64 bytes (wMaxTransferSize), the CDC code would submit an OUT transfer with N<64 bytes length to fill the buffer back up. However if the host had more than N bytes to send then it would still send the full 64 bytes (correctly) in the transfer. The remaining (64-N) bytes would be lost. Adds the restriction that CDCInterface rxbuf has to be at least 64 bytes. Fixes issue #885. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- micropython/usb/usb-device-cdc/manifest.py | 2 +- micropython/usb/usb-device-cdc/usb/device/cdc.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/micropython/usb/usb-device-cdc/manifest.py b/micropython/usb/usb-device-cdc/manifest.py index af9b8cb84..4520325e3 100644 --- a/micropython/usb/usb-device-cdc/manifest.py +++ b/micropython/usb/usb-device-cdc/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.0") +metadata(version="0.1.1") require("usb-device") package("usb") diff --git a/micropython/usb/usb-device-cdc/usb/device/cdc.py b/micropython/usb/usb-device-cdc/usb/device/cdc.py index 741eaafb2..28bfb0657 100644 --- a/micropython/usb/usb-device-cdc/usb/device/cdc.py +++ b/micropython/usb/usb-device-cdc/usb/device/cdc.py @@ -144,8 +144,8 @@ def init( if flow != 0: raise NotImplementedError # UART flow control currently not supported - if not (txbuf and rxbuf): - raise ValueError # Buffer sizes are required + if not (txbuf and rxbuf >= _BULK_EP_LEN): + raise ValueError # Buffer sizes are required, rxbuf must be at least one EP self._timeout = timeout self._wb = Buffer(txbuf) @@ -330,7 +330,11 @@ def _wr_cb(self, ep, res, num_bytes): def _rd_xfer(self): # Keep an active data OUT transfer to read data from the host, # whenever the receive buffer has room for new data - if self.is_open() and not self.xfer_pending(self.ep_d_out) and self._rb.writable(): + if ( + self.is_open() + and not self.xfer_pending(self.ep_d_out) + and self._rb.writable() >= _BULK_EP_LEN + ): # Can only submit up to the endpoint length per transaction, otherwise we won't # get any transfer callback until the full transaction completes. self.submit_xfer(self.ep_d_out, self._rb.pend_write(_BULK_EP_LEN), self._rd_cb) From fbf7e120c6830d8d04097309e715bcab63dcca67 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 3 Jul 2024 17:18:49 +1000 Subject: [PATCH 34/88] usb-device-keyboard: Fix ; and ` keycode names. They should be named as the un-shifted version. Signed-off-by: Damien George --- micropython/usb/usb-device-keyboard/manifest.py | 2 +- micropython/usb/usb-device-keyboard/usb/device/keyboard.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/micropython/usb/usb-device-keyboard/manifest.py b/micropython/usb/usb-device-keyboard/manifest.py index 923535c4c..5a2ff307d 100644 --- a/micropython/usb/usb-device-keyboard/manifest.py +++ b/micropython/usb/usb-device-keyboard/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.0") +metadata(version="0.1.1") require("usb-device-hid") package("usb") diff --git a/micropython/usb/usb-device-keyboard/usb/device/keyboard.py b/micropython/usb/usb-device-keyboard/usb/device/keyboard.py index c42405fc4..22091c50b 100644 --- a/micropython/usb/usb-device-keyboard/usb/device/keyboard.py +++ b/micropython/usb/usb-device-keyboard/usb/device/keyboard.py @@ -163,9 +163,9 @@ class KeyCode: CLOSE_BRACKET = 48 # ] } BACKSLASH = 49 # \ | HASH = 50 # # ~ - COLON = 51 # ; : + SEMICOLON = 51 # ; : QUOTE = 52 # ' " - TILDE = 53 # ` ~ + GRAVE = 53 # ` ~ COMMA = 54 # , < DOT = 55 # . > SLASH = 56 # / ? From 60d137029f169efde062464970cdee6f7367abb8 Mon Sep 17 00:00:00 2001 From: Max Holliday Date: Mon, 8 Jul 2024 20:01:02 -0700 Subject: [PATCH 35/88] lora-sx126x: Change to class-level memoryview for _cmd buf. Currently, the LoRa SX126x driver dynamically creates at least one, sometimes two, memoryview objects with each call to `_cmd`. This commit simply provides the class with a long-lived memoryview object for `_cmd` to easily slice as necessary. Unlike the SX127x chips, Semtech unfortunately designed the SX126x modems to be more command-centric (as opposed to directly setting registers). Given the amount `_cmd` is called during normal device operation, even a minor improvement here should have a decent impact. Basic TX and RX tests pass on hardware. Signed-off-by: Max Holliday --- micropython/lora/lora-sx126x/lora/sx126x.py | 12 ++++++------ micropython/lora/lora-sx126x/manifest.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/micropython/lora/lora-sx126x/lora/sx126x.py b/micropython/lora/lora-sx126x/lora/sx126x.py index eeb3bffb7..77052c97c 100644 --- a/micropython/lora/lora-sx126x/lora/sx126x.py +++ b/micropython/lora/lora-sx126x/lora/sx126x.py @@ -152,7 +152,7 @@ def __init__( dio3_tcxo_start_time_us if dio3_tcxo_millivolts else 0 ) - self._buf = bytearray(9) # shared buffer for commands + self._buf_view = memoryview(bytearray(9)) # shared buffer for commands # These settings are kept in the object (as can't read them back from the modem) self._output_power = 14 @@ -704,11 +704,11 @@ def _cmd(self, fmt, *write_args, n_read=0, write_buf=None, read_buf=None): # have happened well before _cmd() is called again. self._wait_not_busy(self._busy_timeout) - # Pack write_args into _buf and wrap a memoryview of the correct length around it + # Pack write_args into slice of _buf_view memoryview of correct length wrlen = struct.calcsize(fmt) - assert n_read + wrlen <= len(self._buf) # if this fails, make _buf bigger! - struct.pack_into(fmt, self._buf, 0, *write_args) - buf = memoryview(self._buf)[: (wrlen + n_read)] + assert n_read + wrlen <= len(self._buf_view) # if this fails, make _buf bigger! + struct.pack_into(fmt, self._buf_view, 0, *write_args) + buf = self._buf_view[: (wrlen + n_read)] if _DEBUG: print(">>> {}".format(buf[:wrlen].hex())) @@ -723,7 +723,7 @@ def _cmd(self, fmt, *write_args, n_read=0, write_buf=None, read_buf=None): self._cs(1) if n_read > 0: - res = memoryview(buf)[wrlen : (wrlen + n_read)] # noqa: E203 + res = self._buf_view[wrlen : (wrlen + n_read)] # noqa: E203 if _DEBUG: print("<<< {}".format(res.hex())) return res diff --git a/micropython/lora/lora-sx126x/manifest.py b/micropython/lora/lora-sx126x/manifest.py index 177877091..785a975aa 100644 --- a/micropython/lora/lora-sx126x/manifest.py +++ b/micropython/lora/lora-sx126x/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.2") +metadata(version="0.1.3") require("lora") package("lora") From 910af1889cbb992b63e6de769d1b241375582334 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 20 Aug 2024 16:32:43 +1000 Subject: [PATCH 36/88] tools/build.py: Add "path" entry to index.json. This points to the package's base directory of the within the micropython-lib directory structure. Signed-off-by: Damien George --- tools/build.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/tools/build.py b/tools/build.py index ca664175f..442cf2121 100755 --- a/tools/build.py +++ b/tools/build.py @@ -64,7 +64,7 @@ # index.json is: # { -# "v": 1, <-- file format version +# "v": 2, <-- file format version # "updated": , # "packages": { # { @@ -78,7 +78,9 @@ # "7": ["0.2", "0.3", "0.4"], # ... <-- Other bytecode versions # "py": ["0.1", "0.2", "0.3", "0.4"] -# } +# }, +# // The following entries were added in file format version 2. +# path: "micropython/bluetooth/aioble", # }, # ... # } @@ -122,7 +124,7 @@ import time -_JSON_VERSION_INDEX = 1 +_JSON_VERSION_INDEX = 2 _JSON_VERSION_PACKAGE = 1 @@ -268,7 +270,7 @@ def _copy_as_py( # Update to the latest metadata, and add any new versions to the package in # the index json. -def _update_index_package_metadata(index_package_json, metadata, mpy_version): +def _update_index_package_metadata(index_package_json, metadata, mpy_version, package_path): index_package_json["version"] = metadata.version or "" index_package_json["author"] = "" # TODO: Make manifestfile.py capture this. index_package_json["description"] = metadata.description or "" @@ -283,6 +285,9 @@ def _update_index_package_metadata(index_package_json, metadata, mpy_version): print(" New version {}={}".format(v, metadata.version)) index_package_json["versions"][v].append(metadata.version) + # The following entries were added in file format version 2. + index_package_json["path"] = package_path + def build(output_path, hash_prefix_len, mpy_cross_path): import manifestfile @@ -318,7 +323,8 @@ def build(output_path, hash_prefix_len, mpy_cross_path): for lib_dir in lib_dirs: for manifest_path in glob.glob(os.path.join(lib_dir, "**", "manifest.py"), recursive=True): - print("{}".format(os.path.dirname(manifest_path))) + package_path = os.path.dirname(manifest_path) + print("{}".format(package_path)) # .../foo/manifest.py -> foo package_name = os.path.basename(os.path.dirname(manifest_path)) @@ -342,7 +348,9 @@ def build(output_path, hash_prefix_len, mpy_cross_path): } index_json["packages"].append(index_package_json) - _update_index_package_metadata(index_package_json, manifest.metadata(), mpy_version) + _update_index_package_metadata( + index_package_json, manifest.metadata(), mpy_version, package_path + ) # This is the package json that mip/mpremote downloads. mpy_package_json = { From 8d6ebf57a2e9e610cb11edcd9d3704505e091ba5 Mon Sep 17 00:00:00 2001 From: Robert Klink Date: Wed, 31 Jul 2024 13:58:54 +0200 Subject: [PATCH 37/88] unix-ffi/sqlite3: Fix bytes to accommodate for different pointer sizes. Currently, the bytes object used to store the sqlite3 database pointer is always 4 bytes, which causes segfaults on 64 bit platforms with 8 byte pointers. To address this, the size is now dynamically determined using the uctypes modules pointer size. Signed-off-by: Robert Klink --- unix-ffi/sqlite3/sqlite3.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/unix-ffi/sqlite3/sqlite3.py b/unix-ffi/sqlite3/sqlite3.py index 0f00ff508..1f8bdd6c9 100644 --- a/unix-ffi/sqlite3/sqlite3.py +++ b/unix-ffi/sqlite3/sqlite3.py @@ -1,5 +1,6 @@ import sys import ffilib +import uctypes sq3 = ffilib.open("libsqlite3") @@ -61,6 +62,10 @@ def check_error(db, s): raise Error(s, sqlite3_errmsg(db)) +def get_ptr_size(): + return uctypes.sizeof({"ptr": (0 | uctypes.PTR, uctypes.PTR)}) + + class Connections: def __init__(self, h): self.h = h @@ -83,13 +88,13 @@ def execute(self, sql, params=None): params = [quote(v) for v in params] sql = sql % tuple(params) print(sql) - b = bytearray(4) - s = sqlite3_prepare(self.h, sql, -1, b, None) - check_error(self.h, s) - self.stmnt = int.from_bytes(b, sys.byteorder) - # print("stmnt", self.stmnt) + + stmnt_ptr = bytes(get_ptr_size()) + res = sqlite3_prepare(self.h, sql, -1, stmnt_ptr, None) + check_error(self.h, res) + self.stmnt = int.from_bytes(stmnt_ptr, sys.byteorder) self.num_cols = sqlite3_column_count(self.stmnt) - # print("num_cols", self.num_cols) + # If it's not select, actually execute it here # num_cols == 0 for statements which don't return data (=> modify it) if not self.num_cols: @@ -127,10 +132,9 @@ def fetchone(self): def connect(fname): - b = bytearray(4) - sqlite3_open(fname, b) - h = int.from_bytes(b, sys.byteorder) - return Connections(h) + sqlite_ptr = bytes(get_ptr_size()) + sqlite3_open(fname, sqlite_ptr) + return Connections(int.from_bytes(sqlite_ptr, sys.byteorder)) def quote(val): From 0a65c3d34a4f7c6bf5ed88fe78d4b7f24dc71cdd Mon Sep 17 00:00:00 2001 From: Robert Klink Date: Wed, 31 Jul 2024 14:34:41 +0200 Subject: [PATCH 38/88] unix-ffi/sqlite3: Fix statements not being finalized. Currently, statements are only finalized upon a call to Cursor.close(). However, in Cursor.execute() new statements get created without the previous statements being finalized, causing those to get leaked, preventing the database from being closed. The fix addresses this by finalizing the previous statement if it exists. Signed-off-by: Robert Klink --- unix-ffi/sqlite3/sqlite3.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/unix-ffi/sqlite3/sqlite3.py b/unix-ffi/sqlite3/sqlite3.py index 1f8bdd6c9..24175ec17 100644 --- a/unix-ffi/sqlite3/sqlite3.py +++ b/unix-ffi/sqlite3/sqlite3.py @@ -84,6 +84,11 @@ def __init__(self, h): self.stmnt = None def execute(self, sql, params=None): + if self.stmnt: + # If there is an existing statement, finalize that to free it + res = sqlite3_finalize(self.stmnt) + check_error(self.h, res) + if params: params = [quote(v) for v in params] sql = sql % tuple(params) From ab9c5a01b0a62ad97b47ca02caf6bad8eb8f5a42 Mon Sep 17 00:00:00 2001 From: Robert Klink Date: Wed, 31 Jul 2024 14:38:25 +0200 Subject: [PATCH 39/88] unix-ffi/sqlite3: Add optional parameter for URI support. This commit adds the ability to enable URI on the connect, as can be done in the cpython sqlite3 module. URI allows, among other things, to create a shared named in-memory database, which non URI filenames cannot create. Signed-off-by: Robert Klink --- unix-ffi/sqlite3/sqlite3.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/unix-ffi/sqlite3/sqlite3.py b/unix-ffi/sqlite3/sqlite3.py index 24175ec17..9c645200c 100644 --- a/unix-ffi/sqlite3/sqlite3.py +++ b/unix-ffi/sqlite3/sqlite3.py @@ -6,6 +6,8 @@ sq3 = ffilib.open("libsqlite3") sqlite3_open = sq3.func("i", "sqlite3_open", "sp") +# int sqlite3_config(int, ...); +sqlite3_config = sq3.func("i", "sqlite3_config", "ii") # int sqlite3_close(sqlite3*); sqlite3_close = sq3.func("i", "sqlite3_close", "p") # int sqlite3_prepare( @@ -52,6 +54,8 @@ SQLITE_BLOB = 4 SQLITE_NULL = 5 +SQLITE_CONFIG_URI = 17 + class Error(Exception): pass @@ -136,7 +140,9 @@ def fetchone(self): check_error(self.h, res) -def connect(fname): +def connect(fname, uri=False): + sqlite3_config(SQLITE_CONFIG_URI, int(uri)) + sqlite_ptr = bytes(get_ptr_size()) sqlite3_open(fname, sqlite_ptr) return Connections(int.from_bytes(sqlite_ptr, sys.byteorder)) From 83598cdb3c7fd92928d2ac953141fc0f70793e05 Mon Sep 17 00:00:00 2001 From: Robert Klink Date: Thu, 1 Aug 2024 11:28:22 +0200 Subject: [PATCH 40/88] unix-ffi/sqlite3: Change to use close and prepare v2 versions, clean-up. The sqlite3_prepare and sqlite3_close have been changed to use the v2 version. For the prepare this was done as the v1 version is "legacy", and for close the documentation describes the v2 version to be used for "host languages that are garbage collected, and where the order in which destructors are called is arbitrary", which fits here. Some clean-up to comments has also be done, and the tests now also close the Cursor and Connections. Signed-off-by: Robert Klink --- unix-ffi/sqlite3/sqlite3.py | 40 ++++++++++++++++-------------- unix-ffi/sqlite3/test_sqlite3.py | 3 +++ unix-ffi/sqlite3/test_sqlite3_2.py | 3 +++ 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/unix-ffi/sqlite3/sqlite3.py b/unix-ffi/sqlite3/sqlite3.py index 9c645200c..35c0c011e 100644 --- a/unix-ffi/sqlite3/sqlite3.py +++ b/unix-ffi/sqlite3/sqlite3.py @@ -5,11 +5,15 @@ sq3 = ffilib.open("libsqlite3") +# int sqlite3_open( +# const char *filename, /* Database filename (UTF-8) */ +# sqlite3 **ppDb /* OUT: SQLite db handle */ +# ); sqlite3_open = sq3.func("i", "sqlite3_open", "sp") # int sqlite3_config(int, ...); sqlite3_config = sq3.func("i", "sqlite3_config", "ii") -# int sqlite3_close(sqlite3*); -sqlite3_close = sq3.func("i", "sqlite3_close", "p") +# int sqlite3_close_v2(sqlite3*); +sqlite3_close = sq3.func("i", "sqlite3_close_v2", "p") # int sqlite3_prepare( # sqlite3 *db, /* Database handle */ # const char *zSql, /* SQL statement, UTF-8 encoded */ @@ -17,7 +21,7 @@ # sqlite3_stmt **ppStmt, /* OUT: Statement handle */ # const char **pzTail /* OUT: Pointer to unused portion of zSql */ # ); -sqlite3_prepare = sq3.func("i", "sqlite3_prepare", "psipp") +sqlite3_prepare = sq3.func("i", "sqlite3_prepare_v2", "psipp") # int sqlite3_finalize(sqlite3_stmt *pStmt); sqlite3_finalize = sq3.func("i", "sqlite3_finalize", "p") # int sqlite3_step(sqlite3_stmt*); @@ -26,20 +30,17 @@ sqlite3_column_count = sq3.func("i", "sqlite3_column_count", "p") # int sqlite3_column_type(sqlite3_stmt*, int iCol); sqlite3_column_type = sq3.func("i", "sqlite3_column_type", "pi") +# int sqlite3_column_int(sqlite3_stmt*, int iCol); sqlite3_column_int = sq3.func("i", "sqlite3_column_int", "pi") -# using "d" return type gives wrong results +# double sqlite3_column_double(sqlite3_stmt*, int iCol); sqlite3_column_double = sq3.func("d", "sqlite3_column_double", "pi") +# const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); sqlite3_column_text = sq3.func("s", "sqlite3_column_text", "pi") # sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); -# TODO: should return long int -sqlite3_last_insert_rowid = sq3.func("i", "sqlite3_last_insert_rowid", "p") +sqlite3_last_insert_rowid = sq3.func("l", "sqlite3_last_insert_rowid", "p") # const char *sqlite3_errmsg(sqlite3*); sqlite3_errmsg = sq3.func("s", "sqlite3_errmsg", "p") -# Too recent -##const char *sqlite3_errstr(int); -# sqlite3_errstr = sq3.func("s", "sqlite3_errstr", "i") - SQLITE_OK = 0 SQLITE_ERROR = 1 @@ -78,8 +79,10 @@ def cursor(self): return Cursor(self.h) def close(self): - s = sqlite3_close(self.h) - check_error(self.h, s) + if self.h: + s = sqlite3_close(self.h) + check_error(self.h, s) + self.h = None class Cursor: @@ -96,7 +99,6 @@ def execute(self, sql, params=None): if params: params = [quote(v) for v in params] sql = sql % tuple(params) - print(sql) stmnt_ptr = bytes(get_ptr_size()) res = sqlite3_prepare(self.h, sql, -1, stmnt_ptr, None) @@ -104,22 +106,23 @@ def execute(self, sql, params=None): self.stmnt = int.from_bytes(stmnt_ptr, sys.byteorder) self.num_cols = sqlite3_column_count(self.stmnt) - # If it's not select, actually execute it here - # num_cols == 0 for statements which don't return data (=> modify it) if not self.num_cols: v = self.fetchone() + # If it's not select, actually execute it here + # num_cols == 0 for statements which don't return data (=> modify it) assert v is None self.lastrowid = sqlite3_last_insert_rowid(self.h) def close(self): - s = sqlite3_finalize(self.stmnt) - check_error(self.h, s) + if self.stmnt: + s = sqlite3_finalize(self.stmnt) + check_error(self.h, s) + self.stmnt = None def make_row(self): res = [] for i in range(self.num_cols): t = sqlite3_column_type(self.stmnt, i) - # print("type", t) if t == SQLITE_INTEGER: res.append(sqlite3_column_int(self.stmnt, i)) elif t == SQLITE_FLOAT: @@ -132,7 +135,6 @@ def make_row(self): def fetchone(self): res = sqlite3_step(self.stmnt) - # print("step:", res) if res == SQLITE_DONE: return None if res == SQLITE_ROW: diff --git a/unix-ffi/sqlite3/test_sqlite3.py b/unix-ffi/sqlite3/test_sqlite3.py index 39dc07549..b168f18ff 100644 --- a/unix-ffi/sqlite3/test_sqlite3.py +++ b/unix-ffi/sqlite3/test_sqlite3.py @@ -17,3 +17,6 @@ assert row == e assert expected == [] + +cur.close() +conn.close() diff --git a/unix-ffi/sqlite3/test_sqlite3_2.py b/unix-ffi/sqlite3/test_sqlite3_2.py index 68a2abb86..515f865c3 100644 --- a/unix-ffi/sqlite3/test_sqlite3_2.py +++ b/unix-ffi/sqlite3/test_sqlite3_2.py @@ -10,3 +10,6 @@ cur.execute("SELECT * FROM foo") assert cur.fetchone() == (42,) assert cur.fetchone() is None + +cur.close() +conn.close() From b77f67bd7ccd6701e2bf3333a50c56fa709b68fc Mon Sep 17 00:00:00 2001 From: Robert Klink Date: Tue, 6 Aug 2024 15:24:39 +0200 Subject: [PATCH 41/88] unix-ffi/sqlite3: Add commit and rollback functionality like CPython. To increase the similarity between this module and CPythons sqlite3 module the commit() and rollback() as defined in CPythons version have been added, along with the different (auto)commit behaviors present there. The defaults are also set to the same as in CPython, and can be changed with the same parameters in connect(), as is showcased in the new test. Signed-off-by: Robert Klink --- unix-ffi/sqlite3/sqlite3.py | 133 ++++++++++++++++++++--------- unix-ffi/sqlite3/test_sqlite3_3.py | 42 +++++++++ 2 files changed, 137 insertions(+), 38 deletions(-) create mode 100644 unix-ffi/sqlite3/test_sqlite3_3.py diff --git a/unix-ffi/sqlite3/sqlite3.py b/unix-ffi/sqlite3/sqlite3.py index 35c0c011e..299f8247d 100644 --- a/unix-ffi/sqlite3/sqlite3.py +++ b/unix-ffi/sqlite3/sqlite3.py @@ -12,6 +12,8 @@ sqlite3_open = sq3.func("i", "sqlite3_open", "sp") # int sqlite3_config(int, ...); sqlite3_config = sq3.func("i", "sqlite3_config", "ii") +# int sqlite3_get_autocommit(sqlite3*); +sqlite3_get_autocommit = sq3.func("i", "sqlite3_get_autocommit", "p") # int sqlite3_close_v2(sqlite3*); sqlite3_close = sq3.func("i", "sqlite3_close_v2", "p") # int sqlite3_prepare( @@ -57,6 +59,9 @@ SQLITE_CONFIG_URI = 17 +# For compatibility with CPython sqlite3 driver +LEGACY_TRANSACTION_CONTROL = -1 + class Error(Exception): pass @@ -71,86 +76,138 @@ def get_ptr_size(): return uctypes.sizeof({"ptr": (0 | uctypes.PTR, uctypes.PTR)}) +def __prepare_stmt(db, sql): + # Prepares a statement + stmt_ptr = bytes(get_ptr_size()) + res = sqlite3_prepare(db, sql, -1, stmt_ptr, None) + check_error(db, res) + return int.from_bytes(stmt_ptr, sys.byteorder) + +def __exec_stmt(db, sql): + # Prepares, executes, and finalizes a statement + stmt = __prepare_stmt(db, sql) + sqlite3_step(stmt) + res = sqlite3_finalize(stmt) + check_error(db, res) + +def __is_dml(sql): + # Checks if a sql query is a DML, as these get a BEGIN in LEGACY_TRANSACTION_CONTROL + for dml in ["INSERT", "DELETE", "UPDATE", "MERGE"]: + if dml in sql.upper(): + return True + return False + + class Connections: - def __init__(self, h): - self.h = h + def __init__(self, db, isolation_level, autocommit): + self.db = db + self.isolation_level = isolation_level + self.autocommit = autocommit + + def commit(self): + if self.autocommit == LEGACY_TRANSACTION_CONTROL and not sqlite3_get_autocommit(self.db): + __exec_stmt(self.db, "COMMIT") + elif self.autocommit == False: + __exec_stmt(self.db, "COMMIT") + __exec_stmt(self.db, "BEGIN") + + def rollback(self): + if self.autocommit == LEGACY_TRANSACTION_CONTROL and not sqlite3_get_autocommit(self.db): + __exec_stmt(self.db, "ROLLBACK") + elif self.autocommit == False: + __exec_stmt(self.db, "ROLLBACK") + __exec_stmt(self.db, "BEGIN") def cursor(self): - return Cursor(self.h) + return Cursor(self.db, self.isolation_level, self.autocommit) def close(self): - if self.h: - s = sqlite3_close(self.h) - check_error(self.h, s) - self.h = None + if self.db: + if self.autocommit == False and not sqlite3_get_autocommit(self.db): + __exec_stmt(self.db, "ROLLBACK") + + res = sqlite3_close(self.db) + check_error(self.db, res) + self.db = None class Cursor: - def __init__(self, h): - self.h = h - self.stmnt = None + def __init__(self, db, isolation_level, autocommit): + self.db = db + self.isolation_level = isolation_level + self.autocommit = autocommit + self.stmt = None + + def __quote(val): + if isinstance(val, str): + return "'%s'" % val + return str(val) def execute(self, sql, params=None): - if self.stmnt: + if self.stmt: # If there is an existing statement, finalize that to free it - res = sqlite3_finalize(self.stmnt) - check_error(self.h, res) + res = sqlite3_finalize(self.stmt) + check_error(self.db, res) if params: - params = [quote(v) for v in params] + params = [self.__quote(v) for v in params] sql = sql % tuple(params) - stmnt_ptr = bytes(get_ptr_size()) - res = sqlite3_prepare(self.h, sql, -1, stmnt_ptr, None) - check_error(self.h, res) - self.stmnt = int.from_bytes(stmnt_ptr, sys.byteorder) - self.num_cols = sqlite3_column_count(self.stmnt) + if __is_dml(sql) and self.autocommit == LEGACY_TRANSACTION_CONTROL and sqlite3_get_autocommit(self.db): + # For compatibility with CPython, add functionality for their default transaction + # behavior. Changing autocommit from LEGACY_TRANSACTION_CONTROL will remove this + __exec_stmt(self.db, "BEGIN " + self.isolation_level) + + self.stmt = __prepare_stmt(self.db, sql) + self.num_cols = sqlite3_column_count(self.stmt) if not self.num_cols: v = self.fetchone() # If it's not select, actually execute it here # num_cols == 0 for statements which don't return data (=> modify it) assert v is None - self.lastrowid = sqlite3_last_insert_rowid(self.h) + self.lastrowid = sqlite3_last_insert_rowid(self.db) def close(self): - if self.stmnt: - s = sqlite3_finalize(self.stmnt) - check_error(self.h, s) - self.stmnt = None + if self.stmt: + res = sqlite3_finalize(self.stmt) + check_error(self.db, res) + self.stmt = None - def make_row(self): + def __make_row(self): res = [] for i in range(self.num_cols): - t = sqlite3_column_type(self.stmnt, i) + t = sqlite3_column_type(self.stmt, i) if t == SQLITE_INTEGER: - res.append(sqlite3_column_int(self.stmnt, i)) + res.append(sqlite3_column_int(self.stmt, i)) elif t == SQLITE_FLOAT: - res.append(sqlite3_column_double(self.stmnt, i)) + res.append(sqlite3_column_double(self.stmt, i)) elif t == SQLITE_TEXT: - res.append(sqlite3_column_text(self.stmnt, i)) + res.append(sqlite3_column_text(self.stmt, i)) else: raise NotImplementedError return tuple(res) def fetchone(self): - res = sqlite3_step(self.stmnt) + res = sqlite3_step(self.stmt) if res == SQLITE_DONE: return None if res == SQLITE_ROW: - return self.make_row() - check_error(self.h, res) + return self.__make_row() + check_error(self.db, res) + +def connect(fname, uri=False, isolation_level="", autocommit=LEGACY_TRANSACTION_CONTROL): + if isolation_level not in [None, "", "DEFERRED", "IMMEDIATE", "EXCLUSIVE"]: + raise Error("Invalid option for isolation level") -def connect(fname, uri=False): sqlite3_config(SQLITE_CONFIG_URI, int(uri)) sqlite_ptr = bytes(get_ptr_size()) sqlite3_open(fname, sqlite_ptr) - return Connections(int.from_bytes(sqlite_ptr, sys.byteorder)) + db = int.from_bytes(sqlite_ptr, sys.byteorder) + if autocommit == False: + __exec_stmt(db, "BEGIN") -def quote(val): - if isinstance(val, str): - return "'%s'" % val - return str(val) + return Connections(db, isolation_level, autocommit) diff --git a/unix-ffi/sqlite3/test_sqlite3_3.py b/unix-ffi/sqlite3/test_sqlite3_3.py new file mode 100644 index 000000000..0a6fefc97 --- /dev/null +++ b/unix-ffi/sqlite3/test_sqlite3_3.py @@ -0,0 +1,42 @@ +import sqlite3 + + +def test_autocommit(): + conn = sqlite3.connect(":memory:", autocommit=True) + + # First cursor creates table and inserts value (DML) + cur = conn.cursor() + cur.execute("CREATE TABLE foo(a int)") + cur.execute("INSERT INTO foo VALUES (42)") + cur.close() + + # Second cursor fetches 42 due to the autocommit + cur = conn.cursor() + cur.execute("SELECT * FROM foo") + assert cur.fetchone() == (42,) + assert cur.fetchone() is None + + cur.close() + conn.close() + +def test_manual(): + conn = sqlite3.connect(":memory:", autocommit=False) + + # First cursor creates table, insert rolls back + cur = conn.cursor() + cur.execute("CREATE TABLE foo(a int)") + conn.commit() + cur.execute("INSERT INTO foo VALUES (42)") + cur.close() + conn.rollback() + + # Second connection fetches nothing due to the rollback + cur = conn.cursor() + cur.execute("SELECT * FROM foo") + assert cur.fetchone() is None + + cur.close() + conn.close() + +test_autocommit() +test_manual() From bea5367ce23d9683d15ff19c16e246449827cb4b Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 22 Aug 2024 13:05:33 +1000 Subject: [PATCH 42/88] unix-ffi/sqlite3: Bump version to 0.3.0. The previous commits fixed bugs and added new features. Signed-off-by: Damien George --- unix-ffi/sqlite3/manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unix-ffi/sqlite3/manifest.py b/unix-ffi/sqlite3/manifest.py index 63cdf4b9f..5b04d71d3 100644 --- a/unix-ffi/sqlite3/manifest.py +++ b/unix-ffi/sqlite3/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.4") +metadata(version="0.3.0") # Originally written by Paul Sokolovsky. From 66fa62bda10d199be5b24457e044cb863d5d216a Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 22 Aug 2024 13:08:29 +1000 Subject: [PATCH 43/88] tools/ci.sh: Add sqlite3 tests to CI. Signed-off-by: Damien George --- tools/ci.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/ci.sh b/tools/ci.sh index c2cf9dbad..07b27d13c 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -66,6 +66,9 @@ function ci_package_tests_run { unix-ffi/gettext/test_gettext.py \ unix-ffi/pwd/test_getpwnam.py \ unix-ffi/re/test_re.py \ + unix-ffi/sqlite3/test_sqlite3.py \ + unix-ffi/sqlite3/test_sqlite3_2.py \ + unix-ffi/sqlite3/test_sqlite3_3.py \ unix-ffi/time/test_strftime.py \ ; do echo "Running test $test" From 1effa11c77c409cbe938cac95a7331e4e1385f9a Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 20 Feb 2024 10:49:42 +1100 Subject: [PATCH 44/88] CONTRIBUTING: Add extra explanation of "Publish packages for branch". I hadn't used this feature for a while, and realised there's one confusing element of it not previously mentioned in the docs. Signed-off-by: Angus Gratton --- CONTRIBUTING.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d590754e9..61a49101e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,15 +102,20 @@ Pages](https://docs.github.com/en/pages): "truthy" value). 5. The settings for GitHub Actions and GitHub Pages features should not need to be changed from the repository defaults, unless you've explicitly disabled - them. + Actions or Pages in your fork. The next time you push commits to a branch in your fork, GitHub Actions will run an additional step in the "Build All Packages" workflow named "Publish Packages -for branch". +for branch". This step runs in *your fork*, but if you open a pull request then +this workflow is not shown in the Pull Request's "Checks". These run in the +upstream repository. Navigate to your fork's Actions tab in order to see +the additional "Publish Packages for branch" step. Anyone can then install these packages as described under [Installing packages -from forks](README.md#installing-packages-from-forks). The exact commands are also -quoted in the GitHub Actions log for the "Publish Packages for branch" step. +from forks](README.md#installing-packages-from-forks). + +The exact command is also quoted in the GitHub Actions log in your fork's +Actions for the "Publish Packages for branch" step of "Build All Packages". #### Opting Back Out From 27e4d73bc2618d378a0610960cf5e81985e5d914 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 27 Aug 2024 12:04:27 +1000 Subject: [PATCH 45/88] umqtt.robust: Remove reference to missing example. It looks like this example file was not added to the original commit back in 6190cec14a6514c4adc8dfdc33b355be10db0400. Fixes issue #320. Signed-off-by: Angus Gratton --- micropython/umqtt.robust/example_sub_robust.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/micropython/umqtt.robust/example_sub_robust.py b/micropython/umqtt.robust/example_sub_robust.py index c991c70a1..f09befe02 100644 --- a/micropython/umqtt.robust/example_sub_robust.py +++ b/micropython/umqtt.robust/example_sub_robust.py @@ -19,8 +19,7 @@ def sub_cb(topic, msg): # # There can be a problem when a session for a given client exists, # but doesn't have subscriptions a particular application expects. -# In this case, a session needs to be cleaned first. See -# example_reset_session.py for an obvious way how to do that. +# In this case, a session needs to be cleaned first. # # In an actual application, it's up to its developer how to # manage these issues. One extreme is to have external "provisioning" From 1d3c722b7d3b4ada25f248166cf056bc9155278f Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 12 Jun 2024 15:42:18 +1000 Subject: [PATCH 46/88] usb: Fix race if transfers are submitted by a thread. The USB pending transfer flag was cleared before calling the completion callback, to allow the callback code to call submit_xfer() again. Unfortunately this isn't safe in a multi-threaded environment, as another thread may see the endpoint is available before the callback is done executing and submit a new transfer. Rather than adding extra locking, specifically treat the transfer as still pending if checked from another thread while the callback is executing. Closes #874 Signed-off-by: Angus Gratton --- micropython/usb/usb-device/manifest.py | 2 +- micropython/usb/usb-device/usb/device/core.py | 36 ++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/micropython/usb/usb-device/manifest.py b/micropython/usb/usb-device/manifest.py index 0dfab932f..27c9aa88a 100644 --- a/micropython/usb/usb-device/manifest.py +++ b/micropython/usb/usb-device/manifest.py @@ -1,2 +1,2 @@ -metadata(version="0.1.0") +metadata(version="0.1.1") package("usb") diff --git a/micropython/usb/usb-device/usb/device/core.py b/micropython/usb/usb-device/usb/device/core.py index 08277b1f4..06f0f33ce 100644 --- a/micropython/usb/usb-device/usb/device/core.py +++ b/micropython/usb/usb-device/usb/device/core.py @@ -8,6 +8,14 @@ import machine import struct +try: + from _thread import get_ident +except ImportError: + + def get_ident(): + return 0 # Placeholder, for no threading support + + _EP_IN_FLAG = const(1 << 7) # USB descriptor types @@ -76,6 +84,8 @@ def __init__(self): self._itfs = {} # Mapping from interface number to interface object, set by init() self._eps = {} # Mapping from endpoint address to interface object, set by _open_cb() self._ep_cbs = {} # Mapping from endpoint address to Optional[xfer callback] + self._cb_thread = None # Thread currently running endpoint callback + self._cb_ep = None # Endpoint number currently running callback self._usbd = machine.USBDevice() # low-level API def init(self, *itfs, **kwargs): @@ -298,7 +308,7 @@ def _submit_xfer(self, ep_addr, data, done_cb=None): # that function for documentation about the possible parameter values. if ep_addr not in self._eps: raise ValueError("ep_addr") - if self._ep_cbs[ep_addr]: + if self._xfer_pending(ep_addr): raise RuntimeError("xfer_pending") # USBDevice callback may be called immediately, before Python execution @@ -308,12 +318,25 @@ def _submit_xfer(self, ep_addr, data, done_cb=None): self._ep_cbs[ep_addr] = done_cb or True return self._usbd.submit_xfer(ep_addr, data) + def _xfer_pending(self, ep_addr): + # Singleton function to return True if transfer is pending on this endpoint. + # + # Generally, drivers should call Interface.xfer_pending() instead. See that + # function for more documentation. + return self._ep_cbs[ep_addr] or (self._cb_ep == ep_addr and self._cb_thread != get_ident()) + def _xfer_cb(self, ep_addr, result, xferred_bytes): # Singleton callback from TinyUSB custom class driver when a transfer completes. cb = self._ep_cbs.get(ep_addr, None) + self._cb_thread = get_ident() + self._cb_ep = ep_addr # Track while callback is running self._ep_cbs[ep_addr] = None - if callable(cb): - cb(ep_addr, result, xferred_bytes) + try: + # For a pending xfer, 'cb' should either a callback function or True (if no callback) + if callable(cb): + cb(ep_addr, result, xferred_bytes) + finally: + self._cb_ep = None def _control_xfer_cb(self, stage, request): # Singleton callback from TinyUSB custom class driver when a control @@ -528,7 +551,12 @@ def xfer_pending(self, ep_addr): # Return True if a transfer is already pending on ep_addr. # # Only one transfer can be submitted at a time. - return _dev and bool(_dev._ep_cbs[ep_addr]) + # + # The transfer is marked pending while a completion callback is running + # for that endpoint, unless this function is called from the callback + # itself. This makes it simple to submit a new transfer from the + # completion callback. + return _dev and _dev._xfer_pending(ep_addr) def submit_xfer(self, ep_addr, data, done_cb=None): # Submit a USB transfer (of any type except control) From 01f45c118f39610d8fcb2064d237b89ec5b81269 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Thu, 11 Jul 2024 11:23:23 +1000 Subject: [PATCH 47/88] usb: Add a note about buffer thread safety. This is to replace a commit which added locking here but caused some other problems. The idea behind the Buffer class is that a single producer can call pend_write() more than once and it's idempotent, however this is very complex to extend across multiple threads. Signed-off-by: Angus Gratton --- micropython/usb/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/micropython/usb/README.md b/micropython/usb/README.md index 342a0a7e0..d4b975d12 100644 --- a/micropython/usb/README.md +++ b/micropython/usb/README.md @@ -134,3 +134,15 @@ USB MIDI devices in MicroPython. The example [midi_example.py](examples/device/midi_example.py) demonstrates how to create a simple MIDI device to send MIDI data to and from the USB host. + +### Limitations + +#### Buffer thread safety + +The internal Buffer class that's used by most of the USB device classes expects data +to be written to it (i.e. sent to the host) by only one thread. Bytes may be +lost from the USB transfers if more than one thread (or a thread and a callback) +try to write to the buffer simultaneously. + +If writing USB data from multiple sources, your code may need to add +synchronisation (i.e. locks). From c61ca51c6743a95fe8dd5b3345dca41355238271 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Wed, 11 Sep 2024 17:18:57 +1000 Subject: [PATCH 48/88] usb: Tidy up the description of TinyUSB callbacks. Signed-off-by: Angus Gratton --- micropython/usb/usb-device/usb/device/core.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/micropython/usb/usb-device/usb/device/core.py b/micropython/usb/usb-device/usb/device/core.py index 06f0f33ce..2d6790798 100644 --- a/micropython/usb/usb-device/usb/device/core.py +++ b/micropython/usb/usb-device/usb/device/core.py @@ -252,8 +252,8 @@ def active(self, *optional_value): return self._usbd.active(*optional_value) def _open_itf_cb(self, desc): - # Singleton callback from TinyUSB custom class driver, when USB host does - # Set Configuration. Called once per interface or IAD. + # Callback from TinyUSB lower layer, when USB host does Set + # Configuration. Called once per interface or IAD. # Note that even if the configuration descriptor contains an IAD, 'desc' # starts from the first interface descriptor in the IAD and not the IAD @@ -291,7 +291,7 @@ def _open_itf_cb(self, desc): itf.on_open() def _reset_cb(self): - # Callback when the USB device is reset by the host + # TinyUSB lower layer callback when the USB device is reset by the host # Allow interfaces to respond to the reset for itf in self._itfs.values(): @@ -302,7 +302,7 @@ def _reset_cb(self): self._ep_cbs = {} def _submit_xfer(self, ep_addr, data, done_cb=None): - # Singleton function to submit a USB transfer (of any type except control). + # Submit a USB transfer (of any type except control) to TinyUSB lower layer. # # Generally, drivers should call Interface.submit_xfer() instead. See # that function for documentation about the possible parameter values. @@ -319,27 +319,31 @@ def _submit_xfer(self, ep_addr, data, done_cb=None): return self._usbd.submit_xfer(ep_addr, data) def _xfer_pending(self, ep_addr): - # Singleton function to return True if transfer is pending on this endpoint. + # Returns True if a transfer is pending on this endpoint. # # Generally, drivers should call Interface.xfer_pending() instead. See that # function for more documentation. return self._ep_cbs[ep_addr] or (self._cb_ep == ep_addr and self._cb_thread != get_ident()) def _xfer_cb(self, ep_addr, result, xferred_bytes): - # Singleton callback from TinyUSB custom class driver when a transfer completes. + # Callback from TinyUSB lower layer when a transfer completes. cb = self._ep_cbs.get(ep_addr, None) self._cb_thread = get_ident() self._cb_ep = ep_addr # Track while callback is running self._ep_cbs[ep_addr] = None + + # In most cases, 'cb' is a callback function for the transfer. Can also be: + # - True (for a transfer with no callback) + # - None (TinyUSB callback arrived for invalid endpoint, or no transfer. + # Generally unlikely, but may happen in transient states.) try: - # For a pending xfer, 'cb' should either a callback function or True (if no callback) if callable(cb): cb(ep_addr, result, xferred_bytes) finally: self._cb_ep = None def _control_xfer_cb(self, stage, request): - # Singleton callback from TinyUSB custom class driver when a control + # Callback from TinyUSB lower layer when a control # transfer is in progress. # # stage determines appropriate responses (possible values From 394cbfc98a333dd1d4db35fb69379c72c30337f3 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 11 Sep 2024 14:03:52 +1000 Subject: [PATCH 49/88] base64: Remove struct dependency from manifest. This base64 library only uses `struct.unpack` which is available in the built-in `struct` module, so no need for the micropython-lib extras. Signed-off-by: Damien George --- python-stdlib/base64/manifest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python-stdlib/base64/manifest.py b/python-stdlib/base64/manifest.py index 613d3bc62..9e1b31751 100644 --- a/python-stdlib/base64/manifest.py +++ b/python-stdlib/base64/manifest.py @@ -1,6 +1,5 @@ -metadata(version="3.3.5") +metadata(version="3.3.6") require("binascii") -require("struct") module("base64.py") From 7f5ac838655862cb19d8a5762a0a1e0b320b480a Mon Sep 17 00:00:00 2001 From: Jatty Andriean Date: Thu, 4 Jul 2024 07:28:50 +0000 Subject: [PATCH 50/88] lora-sx127x: Fix configuring the implicit header option in the _SX127x. The `_reg_update` method must be called after updating the implicit header option's bit. Signed-off-by: Jatty Andriean --- micropython/lora/lora-sx127x/lora/sx127x.py | 4 ++-- micropython/lora/lora-sx127x/manifest.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/micropython/lora/lora-sx127x/lora/sx127x.py b/micropython/lora/lora-sx127x/lora/sx127x.py index 0226c9696..9faa79a4d 100644 --- a/micropython/lora/lora-sx127x/lora/sx127x.py +++ b/micropython/lora/lora-sx127x/lora/sx127x.py @@ -416,13 +416,13 @@ def configure(self, lora_cfg): modem_config1 |= (self._coding_rate - 4) << _MODEM_CONFIG1_CODING_RATE_SHIFT update_mask |= _MODEM_CONFIG1_CODING_RATE_MASK << _MODEM_CONFIG1_CODING_RATE_SHIFT - self._reg_update(_REG_MODEM_CONFIG1, update_mask, modem_config1) - if "implicit_header" in lora_cfg: self._implicit_header = lora_cfg["implicit_header"] modem_config1 |= _flag(_MODEM_CONFIG1_IMPLICIT_HEADER_MODE_ON, self._implicit_header) update_mask |= _MODEM_CONFIG1_IMPLICIT_HEADER_MODE_ON + self._reg_update(_REG_MODEM_CONFIG1, update_mask, modem_config1) + # Update MODEM_CONFIG2, for any fields that changed modem_config2 = 0 update_mask = 0 diff --git a/micropython/lora/lora-sx127x/manifest.py b/micropython/lora/lora-sx127x/manifest.py index 1936a50e4..177877091 100644 --- a/micropython/lora/lora-sx127x/manifest.py +++ b/micropython/lora/lora-sx127x/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.1") +metadata(version="0.1.2") require("lora") package("lora") From a7cd740b64bfea2e841e646cc495738fccb92950 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20D=C3=B6rre?= Date: Sat, 6 Jul 2024 20:46:51 +0000 Subject: [PATCH 51/88] usb-device: Allow signaling capability of remote_wakeup. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To use this feature you need to create a usb device signaling remote wakeup and then enable remote wakeup on the host (on linux write enabled to /sys/bus/usb/devices//power/wakeup). Then you can wake up the host when is on standby using USBDevice.remote_wakeup. Signed-off-by: Felix Dörre --- micropython/usb/usb-device/manifest.py | 2 +- micropython/usb/usb-device/usb/device/core.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/micropython/usb/usb-device/manifest.py b/micropython/usb/usb-device/manifest.py index 27c9aa88a..025e67547 100644 --- a/micropython/usb/usb-device/manifest.py +++ b/micropython/usb/usb-device/manifest.py @@ -1,2 +1,2 @@ -metadata(version="0.1.1") +metadata(version="0.2.0") package("usb") diff --git a/micropython/usb/usb-device/usb/device/core.py b/micropython/usb/usb-device/usb/device/core.py index 2d6790798..7be09ee46 100644 --- a/micropython/usb/usb-device/usb/device/core.py +++ b/micropython/usb/usb-device/usb/device/core.py @@ -110,6 +110,7 @@ def config( # noqa: PLR0913 device_protocol=0, config_str=None, max_power_ma=None, + remote_wakeup=False, ): # Configure the USB device with a set of interfaces, and optionally reconfiguring the # device and configuration descriptor fields @@ -199,7 +200,7 @@ def maybe_set(value, idx): bmAttributes = ( (1 << 7) # Reserved | (0 if max_power_ma else (1 << 6)) # Self-Powered - # Remote Wakeup not currently supported + | ((1 << 5) if remote_wakeup else 0) ) # Configuration string is optional but supported From 68e3e07bc7ab63931cead3854b2a114e9a084248 Mon Sep 17 00:00:00 2001 From: Joris van der Wel Date: Fri, 2 Aug 2024 20:12:55 +0200 Subject: [PATCH 52/88] aioble: Pass additional connection arguments to gap_connect. This allows the following arguments to be passed to `device.connect()`: * scan_duration_ms * min_conn_interval_us * max_conn_interval_us These are passed as-is to `gap_connect()`. The default value for all of these is `None`, which causes gap_connect to use its own defaults. Signed-off-by: Joris van der Wel --- micropython/bluetooth/aioble-central/manifest.py | 2 +- micropython/bluetooth/aioble-core/manifest.py | 2 +- micropython/bluetooth/aioble/aioble/central.py | 12 ++++++++++-- micropython/bluetooth/aioble/aioble/device.py | 16 ++++++++++++++-- micropython/bluetooth/aioble/manifest.py | 2 +- 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/micropython/bluetooth/aioble-central/manifest.py b/micropython/bluetooth/aioble-central/manifest.py index 9564ecf77..ed61ec9d7 100644 --- a/micropython/bluetooth/aioble-central/manifest.py +++ b/micropython/bluetooth/aioble-central/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.2") +metadata(version="0.3.0") require("aioble-core") diff --git a/micropython/bluetooth/aioble-core/manifest.py b/micropython/bluetooth/aioble-core/manifest.py index c2d335b5c..e040f1076 100644 --- a/micropython/bluetooth/aioble-core/manifest.py +++ b/micropython/bluetooth/aioble-core/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.3.0") +metadata(version="0.4.0") package( "aioble", diff --git a/micropython/bluetooth/aioble/aioble/central.py b/micropython/bluetooth/aioble/aioble/central.py index 6d90cd0f8..131b1e0db 100644 --- a/micropython/bluetooth/aioble/aioble/central.py +++ b/micropython/bluetooth/aioble/aioble/central.py @@ -104,7 +104,9 @@ async def _cancel_pending(): # Start connecting to a peripheral. # Call device.connect() rather than using method directly. -async def _connect(connection, timeout_ms): +async def _connect( + connection, timeout_ms, scan_duration_ms, min_conn_interval_us, max_conn_interval_us +): device = connection.device if device in _connecting: return @@ -122,7 +124,13 @@ async def _connect(connection, timeout_ms): try: with DeviceTimeout(None, timeout_ms): - ble.gap_connect(device.addr_type, device.addr) + ble.gap_connect( + device.addr_type, + device.addr, + scan_duration_ms, + min_conn_interval_us, + max_conn_interval_us, + ) # Wait for the connected IRQ. await connection._event.wait() diff --git a/micropython/bluetooth/aioble/aioble/device.py b/micropython/bluetooth/aioble/aioble/device.py index d02d6385f..93819bc1e 100644 --- a/micropython/bluetooth/aioble/aioble/device.py +++ b/micropython/bluetooth/aioble/aioble/device.py @@ -132,14 +132,26 @@ def __str__(self): def addr_hex(self): return binascii.hexlify(self.addr, ":").decode() - async def connect(self, timeout_ms=10000): + async def connect( + self, + timeout_ms=10000, + scan_duration_ms=None, + min_conn_interval_us=None, + max_conn_interval_us=None, + ): if self._connection: return self._connection # Forward to implementation in central.py. from .central import _connect - await _connect(DeviceConnection(self), timeout_ms) + await _connect( + DeviceConnection(self), + timeout_ms, + scan_duration_ms, + min_conn_interval_us, + max_conn_interval_us, + ) # Start the device task that will clean up after disconnection. self._connection._run_task() diff --git a/micropython/bluetooth/aioble/manifest.py b/micropython/bluetooth/aioble/manifest.py index 824647429..832200570 100644 --- a/micropython/bluetooth/aioble/manifest.py +++ b/micropython/bluetooth/aioble/manifest.py @@ -3,7 +3,7 @@ # code. This allows (for development purposes) all the files to live in the # one directory. -metadata(version="0.5.2") +metadata(version="0.6.0") # Default installation gives you everything. Install the individual # components (or a combination of them) if you want a more minimal install. From d6faaf847243766c1bca92d8bca603025d3f19ba Mon Sep 17 00:00:00 2001 From: Prabhu Ullagaddi Date: Wed, 6 Nov 2024 17:58:20 +0530 Subject: [PATCH 53/88] umqtt.simple: Add optional socket timeout to connect method. If there are any network issues, mqtt will block on the socket non-deterministically. This commit introduces a `timeout` option which can be used to set a finite timeout on the socket. Upon any issue, mqtth lib will throw exception. --- micropython/umqtt.simple/manifest.py | 2 +- micropython/umqtt.simple/umqtt/simple.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/micropython/umqtt.simple/manifest.py b/micropython/umqtt.simple/manifest.py index b418995c5..45f9edfbd 100644 --- a/micropython/umqtt.simple/manifest.py +++ b/micropython/umqtt.simple/manifest.py @@ -1,4 +1,4 @@ -metadata(description="Lightweight MQTT client for MicroPython.", version="1.4.0") +metadata(description="Lightweight MQTT client for MicroPython.", version="1.5.0") # Originally written by Paul Sokolovsky. diff --git a/micropython/umqtt.simple/umqtt/simple.py b/micropython/umqtt.simple/umqtt/simple.py index 6da38e445..2a5b91655 100644 --- a/micropython/umqtt.simple/umqtt/simple.py +++ b/micropython/umqtt.simple/umqtt/simple.py @@ -60,8 +60,9 @@ def set_last_will(self, topic, msg, retain=False, qos=0): self.lw_qos = qos self.lw_retain = retain - def connect(self, clean_session=True): + def connect(self, clean_session=True, timeout=None): self.sock = socket.socket() + self.sock.settimeout(timeout) addr = socket.getaddrinfo(self.server, self.port)[0][-1] self.sock.connect(addr) if self.ssl: From a0ceed82695b038f166165118d7621ff6f8ac2c3 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Tue, 5 Nov 2024 11:20:03 +1100 Subject: [PATCH 54/88] aioespnow,webrepl: Use recommended network.WLAN.IF_[AP|STA] constants. Removes the deprecated network.[AP|STA]_IF form. This work was funded through GitHub Sponsors. Signed-off-by: Angus Gratton --- micropython/aioespnow/README.md | 2 +- micropython/net/webrepl/webrepl.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/micropython/aioespnow/README.md b/micropython/aioespnow/README.md index 132bce103..9774d19c3 100644 --- a/micropython/aioespnow/README.md +++ b/micropython/aioespnow/README.md @@ -55,7 +55,7 @@ A small async server example:: import asyncio # A WLAN interface must be active to send()/recv() - network.WLAN(network.STA_IF).active(True) + network.WLAN(network.WLAN.IF_STA).active(True) e = aioespnow.AIOESPNow() # Returns AIOESPNow enhanced with async support e.active(True) diff --git a/micropython/net/webrepl/webrepl.py b/micropython/net/webrepl/webrepl.py index 48c181968..00da8155c 100644 --- a/micropython/net/webrepl/webrepl.py +++ b/micropython/net/webrepl/webrepl.py @@ -102,7 +102,7 @@ def setup_conn(port, accept_handler): listen_s.listen(1) if accept_handler: listen_s.setsockopt(socket.SOL_SOCKET, 20, accept_handler) - for i in (network.AP_IF, network.STA_IF): + for i in (network.WLAN.IF_AP, network.WLAN.IF_STA): iface = network.WLAN(i) if iface.active(): print("WebREPL server started on http://%s:%d/" % (iface.ifconfig()[0], port)) From 0827a31c07804a861db4ee1f2e7e4876618bc8cb Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 6 Nov 2024 11:12:19 +1100 Subject: [PATCH 55/88] tools/ci.sh: Enable unittest tests. Signed-off-by: Damien George --- .../unittest/tests/{test_exception.py => exception.py} | 2 ++ tools/ci.sh | 2 ++ 2 files changed, 4 insertions(+) rename python-stdlib/unittest/tests/{test_exception.py => exception.py} (66%) diff --git a/python-stdlib/unittest/tests/test_exception.py b/python-stdlib/unittest/tests/exception.py similarity index 66% rename from python-stdlib/unittest/tests/test_exception.py rename to python-stdlib/unittest/tests/exception.py index 470ffdcc2..0e828e226 100644 --- a/python-stdlib/unittest/tests/test_exception.py +++ b/python-stdlib/unittest/tests/exception.py @@ -1,3 +1,5 @@ +# This makes unittest return an error code, so is not named "test_xxx.py". + import unittest diff --git a/tools/ci.sh b/tools/ci.sh index 07b27d13c..a5fcdf22e 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -63,6 +63,7 @@ function ci_package_tests_run { python-stdlib/os-path/test_path.py \ python-stdlib/pickle/test_pickle.py \ python-stdlib/string/test_translate.py \ + python-stdlib/unittest/tests/exception.py \ unix-ffi/gettext/test_gettext.py \ unix-ffi/pwd/test_getpwnam.py \ unix-ffi/re/test_re.py \ @@ -90,6 +91,7 @@ function ci_package_tests_run { python-stdlib/shutil \ python-stdlib/tempfile \ python-stdlib/time \ + python-stdlib/unittest/tests \ python-stdlib/unittest-discover/tests \ ; do (cd $path && $MICROPYTHON -m unittest) From 01047889eb1acf115424fee293e03769f6916393 Mon Sep 17 00:00:00 2001 From: Damien George Date: Wed, 6 Nov 2024 11:12:57 +1100 Subject: [PATCH 56/88] unittest: Allow SkipTest to work within a subTest. Signed-off-by: Damien George --- python-stdlib/unittest/tests/test_subtest.py | 14 ++++++++++++++ python-stdlib/unittest/unittest/__init__.py | 13 +++++++------ 2 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 python-stdlib/unittest/tests/test_subtest.py diff --git a/python-stdlib/unittest/tests/test_subtest.py b/python-stdlib/unittest/tests/test_subtest.py new file mode 100644 index 000000000..324150e27 --- /dev/null +++ b/python-stdlib/unittest/tests/test_subtest.py @@ -0,0 +1,14 @@ +import unittest + + +class Test(unittest.TestCase): + def test_subtest_skip(self): + for i in range(4): + with self.subTest(i=i): + print("sub test", i) + if i == 2: + self.skipTest("skip 2") + + +if __name__ == "__main__": + unittest.main() diff --git a/python-stdlib/unittest/unittest/__init__.py b/python-stdlib/unittest/unittest/__init__.py index f15f30448..6d7fa40b8 100644 --- a/python-stdlib/unittest/unittest/__init__.py +++ b/python-stdlib/unittest/unittest/__init__.py @@ -348,7 +348,13 @@ def _handle_test_exception( exc = exc_info[1] traceback = exc_info[2] ex_str = _capture_exc(exc, traceback) - if isinstance(exc, AssertionError): + if isinstance(exc, SkipTest): + reason = exc.args[0] + test_result.skippedNum += 1 + test_result.skipped.append((current_test, reason)) + print(" skipped:", reason) + return + elif isinstance(exc, AssertionError): test_result.failuresNum += 1 test_result.failures.append((current_test, ex_str)) if verbose: @@ -396,11 +402,6 @@ def run_one(test_function): print(" FAIL") else: print(" ok") - except SkipTest as e: - reason = e.args[0] - print(" skipped:", reason) - test_result.skippedNum += 1 - test_result.skipped.append((name, c, reason)) except Exception as ex: _handle_test_exception( current_test=(name, c), test_result=test_result, exc_info=(type(ex), ex, None) From e4cf09527bce7569f5db742cf6ae9db68d50c6a9 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 18 Nov 2024 12:00:38 +1100 Subject: [PATCH 57/88] unittest: Always use "raise" with an argument. So this code can be compiled with the MicroPython native emitter, which does not support "raise" without any arguments. Signed-off-by: Damien George --- python-stdlib/unittest/manifest.py | 2 +- python-stdlib/unittest/unittest/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python-stdlib/unittest/manifest.py b/python-stdlib/unittest/manifest.py index 101e3e833..a01bbb8e6 100644 --- a/python-stdlib/unittest/manifest.py +++ b/python-stdlib/unittest/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.10.3") +metadata(version="0.10.4") package("unittest") diff --git a/python-stdlib/unittest/unittest/__init__.py b/python-stdlib/unittest/unittest/__init__.py index 6d7fa40b8..61b315788 100644 --- a/python-stdlib/unittest/unittest/__init__.py +++ b/python-stdlib/unittest/unittest/__init__.py @@ -198,7 +198,7 @@ def assertRaises(self, exc, func=None, *args, **kwargs): except Exception as e: if isinstance(e, exc): return - raise + raise e assert False, "%r not raised" % exc @@ -407,7 +407,7 @@ def run_one(test_function): current_test=(name, c), test_result=test_result, exc_info=(type(ex), ex, None) ) # Uncomment to investigate failure in detail - # raise + # raise ex finally: __test_result__ = None __current_test__ = None From 65a14116d503dd544cbba92284517c99e5e9a448 Mon Sep 17 00:00:00 2001 From: Richard Weickelt Date: Wed, 11 Dec 2024 22:04:43 +0100 Subject: [PATCH 58/88] requests: Do not leak header modifications when calling request. The requests() function takes a headers dict argument (call-by-reference). This object is then modified in the function. For instance the host is added and authentication information. Such behavior is not expected. It is also problematic: - Modifications of the header dictionary will be visible on the caller site. - When reusing the same (supposedly read-only) headers object for differenct calls, the second call will apparently re-use wrong headers from the previous call and may fail. This patch should also fix #839. Unfortunately the copy operation does not preserve the key order and we have to touch the existing test cases. Signed-off-by: Richard Weickelt --- python-ecosys/requests/requests/__init__.py | 2 ++ python-ecosys/requests/test_requests.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index a9a183619..2951035f7 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -46,6 +46,8 @@ def request( ): if headers is None: headers = {} + else: + headers = headers.copy() redirect = None # redirection url, None means no redirection chunked_data = data and getattr(data, "__next__", None) and not getattr(data, "__len__", None) diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py index 513e533a3..ac77291b0 100644 --- a/python-ecosys/requests/test_requests.py +++ b/python-ecosys/requests/test_requests.py @@ -102,11 +102,11 @@ def chunks(): def test_overwrite_get_headers(): response = requests.request( - "GET", "http://example.com/", headers={"Connection": "keep-alive", "Host": "test.com"} + "GET", "http://example.com/", headers={"Host": "test.com", "Connection": "keep-alive"} ) assert response.raw._write_buffer.getvalue() == ( - b"GET / HTTP/1.0\r\n" + b"Host: test.com\r\n" + b"Connection: keep-alive\r\n\r\n" + b"GET / HTTP/1.0\r\n" + b"Connection: keep-alive\r\n" + b"Host: test.com\r\n\r\n" ), format_message(response) @@ -145,6 +145,14 @@ def chunks(): ), format_message(response) +def test_do_not_modify_headers_argument(): + global do_not_modify_this_dict + do_not_modify_this_dict = {} + requests.request("GET", "http://example.com/", headers=do_not_modify_this_dict) + + assert do_not_modify_this_dict == {}, do_not_modify_this_dict + + test_simple_get() test_get_auth() test_get_custom_header() @@ -153,3 +161,4 @@ def chunks(): test_overwrite_get_headers() test_overwrite_post_json_headers() test_overwrite_post_chunked_data_headers() +test_do_not_modify_headers_argument() From 7a32df3d133b89bf989b28b5fad3c9c118a9b7ed Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 24 Feb 2025 14:20:16 +1100 Subject: [PATCH 59/88] requests: Bump version to 0.10.1. The previous commit fixed a bug. Signed-off-by: Damien George --- python-ecosys/requests/manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index eb7bb2d42..f8343e2a1 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.10.0", pypi="requests") +metadata(version="0.10.1", pypi="requests") package("requests") From b379e4fb4cb2b014be21fba698247ab81fc1c17a Mon Sep 17 00:00:00 2001 From: Glenn Moloney Date: Tue, 18 Feb 2025 11:05:50 +1100 Subject: [PATCH 60/88] mip: Allow relative URLs in package.json. This allows to specify relative URLs in package.json, which are resolved relative to the package.json URL. This mirrors the functionality added to mpremote in https://github.com/micropython/micropython/pull/12477. Signed-off-by: Glenn Moloney --- micropython/mip/manifest.py | 2 +- micropython/mip/mip/__init__.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/micropython/mip/manifest.py b/micropython/mip/manifest.py index 88fb08da1..2a35f8c5b 100644 --- a/micropython/mip/manifest.py +++ b/micropython/mip/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.3.0", description="On-device package installer for network-capable boards") +metadata(version="0.4.0", description="On-device package installer for network-capable boards") require("requests") diff --git a/micropython/mip/mip/__init__.py b/micropython/mip/mip/__init__.py index 0c3c6f204..8920ad8f4 100644 --- a/micropython/mip/mip/__init__.py +++ b/micropython/mip/mip/__init__.py @@ -9,6 +9,8 @@ _PACKAGE_INDEX = const("https://micropython.org/pi/v2") _CHUNK_SIZE = 128 +allowed_mip_url_prefixes = ("http://", "https://", "github:", "gitlab:") + # This implements os.makedirs(os.dirname(path)) def _ensure_path_exists(path): @@ -124,8 +126,12 @@ def _install_json(package_json_url, index, target, version, mpy): if not _download_file(file_url, fs_target_path): print("File not found: {} {}".format(target_path, short_hash)) return False + base_url = package_json_url.rpartition("/")[0] for target_path, url in package_json.get("urls", ()): fs_target_path = target + "/" + target_path + is_full_url = any(url.startswith(p) for p in allowed_mip_url_prefixes) + if base_url and not is_full_url: + url = f"{base_url}/{url}" # Relative URLs if not _download_file(_rewrite_url(http://github.com/url,%20version), fs_target_path): print("File not found: {} {}".format(target_path, url)) return False @@ -136,12 +142,7 @@ def _install_json(package_json_url, index, target, version, mpy): def _install_package(package, index, target, version, mpy): - if ( - package.startswith("http://") - or package.startswith("https://") - or package.startswith("github:") - or package.startswith("gitlab:") - ): + if any(package.startswith(p) for p in allowed_mip_url_prefixes): if package.endswith(".py") or package.endswith(".mpy"): print("Downloading {} to {}".format(package, target)) return _download_file( From 7337e0802a20bc8394faaf32bb97c60210b6e942 Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 24 Feb 2025 14:27:31 +1100 Subject: [PATCH 61/88] github/workflows: Update actions/upload-artifact to v4. Because v3 is now deprecated. Signed-off-by: Damien George --- .github/workflows/build_packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_packages.yml b/.github/workflows/build_packages.yml index 8854a7307..a89658e2f 100644 --- a/.github/workflows/build_packages.yml +++ b/.github/workflows/build_packages.yml @@ -23,7 +23,7 @@ jobs: if: vars.MICROPY_PUBLISH_MIP_INDEX && github.event_name == 'push' && ! github.event.deleted run: source tools/ci.sh && ci_push_package_index - name: Upload packages as artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: packages-${{ github.sha }} path: ${{ env.PACKAGE_INDEX_PATH }} From 96e17b65d128cac2328e10f317241d3a64aca2c7 Mon Sep 17 00:00:00 2001 From: Damien George Date: Tue, 11 Mar 2025 16:27:09 +1100 Subject: [PATCH 62/88] mip: Make mip.install() skip /rom*/lib directories. If a ROMFS is mounted then "/rom/lib" is usually in `sys.path` before the writable filesystem's "lib" entry. The ROMFS directory cannot be installed to, so skip it if found. Signed-off-by: Damien George --- micropython/mip/manifest.py | 2 +- micropython/mip/mip/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/micropython/mip/manifest.py b/micropython/mip/manifest.py index 2a35f8c5b..9fb94ebcb 100644 --- a/micropython/mip/manifest.py +++ b/micropython/mip/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.4.0", description="On-device package installer for network-capable boards") +metadata(version="0.4.1", description="On-device package installer for network-capable boards") require("requests") diff --git a/micropython/mip/mip/__init__.py b/micropython/mip/mip/__init__.py index 8920ad8f4..7c0fb4d3a 100644 --- a/micropython/mip/mip/__init__.py +++ b/micropython/mip/mip/__init__.py @@ -171,7 +171,7 @@ def _install_package(package, index, target, version, mpy): def install(package, index=None, target=None, version=None, mpy=True): if not target: for p in sys.path: - if p.endswith("/lib"): + if not p.startswith("/rom") and p.endswith("/lib"): target = p break else: From 98d0a2b69a24b9b53309be34d7c5aa6aede45c5e Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 7 Nov 2024 12:25:13 +1100 Subject: [PATCH 63/88] umqtt.simple: Restore legacy ssl/ssl_params arguments. Commit 35d41dbb0e4acf1518f520220d405ebe2db257d6 changed the API for using SSL with umqtt, but only did a minor version increase. This broke various uses of this library, eg https://github.com/aws-samples/aws-iot-core-getting-started-micropython Reinstate the original API for specifying an SSL connection. This library now supports the following: - default, ssl=None or ssl=False: no SSL - ssl=True and optional ssl_params specified: use ssl.wrap_socket - ssl=: use provided SSL context to wrap socket Signed-off-by: Damien George --- micropython/umqtt.simple/manifest.py | 4 +++- micropython/umqtt.simple/umqtt/simple.py | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/micropython/umqtt.simple/manifest.py b/micropython/umqtt.simple/manifest.py index 45f9edfbd..709a27505 100644 --- a/micropython/umqtt.simple/manifest.py +++ b/micropython/umqtt.simple/manifest.py @@ -1,5 +1,7 @@ -metadata(description="Lightweight MQTT client for MicroPython.", version="1.5.0") +metadata(description="Lightweight MQTT client for MicroPython.", version="1.6.0") # Originally written by Paul Sokolovsky. +require("ssl") + package("umqtt") diff --git a/micropython/umqtt.simple/umqtt/simple.py b/micropython/umqtt.simple/umqtt/simple.py index 2a5b91655..d9cdffc47 100644 --- a/micropython/umqtt.simple/umqtt/simple.py +++ b/micropython/umqtt.simple/umqtt/simple.py @@ -17,6 +17,7 @@ def __init__( password=None, keepalive=0, ssl=None, + ssl_params={}, ): if port == 0: port = 8883 if ssl else 1883 @@ -25,6 +26,7 @@ def __init__( self.server = server self.port = port self.ssl = ssl + self.ssl_params = ssl_params self.pid = 0 self.cb = None self.user = user @@ -65,7 +67,12 @@ def connect(self, clean_session=True, timeout=None): self.sock.settimeout(timeout) addr = socket.getaddrinfo(self.server, self.port)[0][-1] self.sock.connect(addr) - if self.ssl: + if self.ssl is True: + # Legacy support for ssl=True and ssl_params arguments. + import ssl + + self.sock = ssl.wrap_socket(self.sock, **self.ssl_params) + elif self.ssl: self.sock = self.ssl.wrap_socket(self.sock, server_hostname=self.server) premsg = bytearray(b"\x10\0\0\0\0\0") msg = bytearray(b"\x04MQTT\x04\x02\0\0") From 3e859d2118c4ef9af5f43c930b3138185e5a9fb4 Mon Sep 17 00:00:00 2001 From: marcsello Date: Mon, 30 Dec 2024 15:20:18 +0100 Subject: [PATCH 64/88] nrf24l01: Increase startup delay. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit According to the datasheet of the NRF240L1 chip, 150 μs startup time is only acceptable when the chip is clocked externally. Most modules use a crystal, which require 1.5 ms to settle. It should be okay to wait more in both cases, for a reliable startup. Signed-off-by: Marcell Pünkösd --- micropython/drivers/radio/nrf24l01/nrf24l01.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropython/drivers/radio/nrf24l01/nrf24l01.py b/micropython/drivers/radio/nrf24l01/nrf24l01.py index 76d55312f..9b034f8ba 100644 --- a/micropython/drivers/radio/nrf24l01/nrf24l01.py +++ b/micropython/drivers/radio/nrf24l01/nrf24l01.py @@ -227,7 +227,7 @@ def send(self, buf, timeout=500): def send_start(self, buf): # power up self.reg_write(CONFIG, (self.reg_read(CONFIG) | PWR_UP) & ~PRIM_RX) - utime.sleep_us(150) + utime.sleep_us(1500) # needs to be 1.5ms # send the data self.cs(0) self.spi.readinto(self.buf, W_TX_PAYLOAD) From bd1ab77324641238e684cd26e1686a890868b096 Mon Sep 17 00:00:00 2001 From: marcsello Date: Mon, 30 Dec 2024 15:47:09 +0100 Subject: [PATCH 65/88] nrf24l01: Properly handle timeout. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The timeout condition was not handled before. Upon timeout, this caused the chip to stay active until another send command changed it's state. Sometimes when it was unable to transmit the data, it got stuck in the tx fifo causing it to fill up over time, which set the TX_FULL flag in the STATUS register. Since there was no exceptions raised, the user code could not differentiate a successful send or a timeout condition. Signed-off-by: Marcell Pünkösd --- micropython/drivers/radio/nrf24l01/nrf24l01.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/micropython/drivers/radio/nrf24l01/nrf24l01.py b/micropython/drivers/radio/nrf24l01/nrf24l01.py index 9b034f8ba..04f07b9d9 100644 --- a/micropython/drivers/radio/nrf24l01/nrf24l01.py +++ b/micropython/drivers/radio/nrf24l01/nrf24l01.py @@ -220,6 +220,13 @@ def send(self, buf, timeout=500): result = None while result is None and utime.ticks_diff(utime.ticks_ms(), start) < timeout: result = self.send_done() # 1 == success, 2 == fail + + if result is None: + # timed out, cancel sending and power down the module + self.flush_tx() + self.reg_write(CONFIG, self.reg_read(CONFIG) & ~PWR_UP) + raise OSError("timed out") + if result == 2: raise OSError("send failed") From c7103bb464507137a32edafc77698e40893b773e Mon Sep 17 00:00:00 2001 From: marcsello Date: Mon, 30 Dec 2024 16:10:49 +0100 Subject: [PATCH 66/88] nrf24l01: Optimize status reading. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The value of the STATUS register is always transmitted by the chip when reading any command. So a R_REGISTER command and the turnaround time can be spared by issuing a NOP command instead. This implementation suggested by the datasheet. This operation is compatible with both nRF24L01 and nRF24L01+. Signed-off-by: Marcell Pünkösd --- micropython/drivers/radio/nrf24l01/nrf24l01.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/micropython/drivers/radio/nrf24l01/nrf24l01.py b/micropython/drivers/radio/nrf24l01/nrf24l01.py index 04f07b9d9..d015250cf 100644 --- a/micropython/drivers/radio/nrf24l01/nrf24l01.py +++ b/micropython/drivers/radio/nrf24l01/nrf24l01.py @@ -130,6 +130,13 @@ def reg_write(self, reg, value): self.cs(1) return ret + def read_status(self): + self.cs(0) + # STATUS register is always shifted during command transmit + self.spi.readinto(self.buf, NOP) + self.cs(1) + return self.buf[0] + def flush_rx(self): self.cs(0) self.spi.readinto(self.buf, FLUSH_RX) @@ -250,7 +257,8 @@ def send_start(self, buf): # returns None if send still in progress, 1 for success, 2 for fail def send_done(self): - if not (self.reg_read(STATUS) & (TX_DS | MAX_RT)): + status = self.read_status() + if not (status & (TX_DS | MAX_RT)): return None # tx not finished # either finished or failed: get and clear status flags, power down From 221a877f8a572a489e9859d0851969d57d71b8d4 Mon Sep 17 00:00:00 2001 From: Damien George Date: Thu, 10 Apr 2025 22:33:53 +1000 Subject: [PATCH 67/88] nrf24l10: Bump minor version. Due to the previous three commits. Signed-off-by: Damien George --- micropython/drivers/radio/nrf24l01/manifest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/micropython/drivers/radio/nrf24l01/manifest.py b/micropython/drivers/radio/nrf24l01/manifest.py index 474d422f9..24276ee4b 100644 --- a/micropython/drivers/radio/nrf24l01/manifest.py +++ b/micropython/drivers/radio/nrf24l01/manifest.py @@ -1,3 +1,3 @@ -metadata(description="nrf24l01 2.4GHz radio driver.", version="0.1.0") +metadata(description="nrf24l01 2.4GHz radio driver.", version="0.2.0") module("nrf24l01.py", opt=3) From f72f3f1a391f15d6fbed01d76f8c97a27427db2f Mon Sep 17 00:00:00 2001 From: Leonard Techel Date: Wed, 29 Jan 2025 10:22:17 +0100 Subject: [PATCH 68/88] lora-sx126x: Fix invert_iq_rx / invert_iq_tx behaviour. This commit fixes a typo and changes a tuple that needs to be mutable to a list (because other parts of the code change elements of this list). Signed-off-by: Damien George --- micropython/lora/lora-sx126x/lora/sx126x.py | 6 +++--- micropython/lora/lora-sx126x/manifest.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/micropython/lora/lora-sx126x/lora/sx126x.py b/micropython/lora/lora-sx126x/lora/sx126x.py index 77052c97c..641367a9f 100644 --- a/micropython/lora/lora-sx126x/lora/sx126x.py +++ b/micropython/lora/lora-sx126x/lora/sx126x.py @@ -363,11 +363,11 @@ def configure(self, lora_cfg): if "preamble_len" in lora_cfg: self._preamble_len = lora_cfg["preamble_len"] - self._invert_iq = ( + self._invert_iq = [ lora_cfg.get("invert_iq_rx", self._invert_iq[0]), lora_cfg.get("invert_iq_tx", self._invert_iq[1]), self._invert_iq[2], - ) + ] if "freq_khz" in lora_cfg: self._rf_freq_hz = int(lora_cfg["freq_khz"] * 1000) @@ -449,7 +449,7 @@ def configure(self, lora_cfg): def _invert_workaround(self, enable): # Apply workaround for DS 15.4 Optimizing the Inverted IQ Operation if self._invert_iq[2] != enable: - val = self._read_read(_REG_IQ_POLARITY_SETUP) + val = self._reg_read(_REG_IQ_POLARITY_SETUP) val = (val & ~4) | _flag(4, enable) self._reg_write(_REG_IQ_POLARITY_SETUP, val) self._invert_iq[2] = enable diff --git a/micropython/lora/lora-sx126x/manifest.py b/micropython/lora/lora-sx126x/manifest.py index 785a975aa..038710820 100644 --- a/micropython/lora/lora-sx126x/manifest.py +++ b/micropython/lora/lora-sx126x/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.3") +metadata(version="0.1.4") require("lora") package("lora") From d1a74360a20dadfaae717d4d8c8bd3531672362f Mon Sep 17 00:00:00 2001 From: Dominik Heidler Date: Fri, 14 Mar 2025 13:36:30 +0100 Subject: [PATCH 69/88] unix-ffi/json: Accept both str and bytes as arg for json.loads(). Same as micropython's internal json lib does. Fixes #985. Signed-off-by: Dominik Heidler --- unix-ffi/json/json/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/unix-ffi/json/json/__init__.py b/unix-ffi/json/json/__init__.py index 69a045563..954618f33 100644 --- a/unix-ffi/json/json/__init__.py +++ b/unix-ffi/json/json/__init__.py @@ -354,8 +354,8 @@ def loads( object_pairs_hook=None, **kw ): - """Deserialize ``s`` (a ``str`` instance containing a JSON - document) to a Python object. + """Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance + containing a JSON document) to a Python object. ``object_hook`` is an optional function that will be called with the result of any object literal decode (a ``dict``). The return value of @@ -413,4 +413,6 @@ def loads( kw["parse_int"] = parse_int if parse_constant is not None: kw["parse_constant"] = parse_constant + if isinstance(s, (bytes, bytearray)): + s = s.decode('utf-8') return cls(**kw).decode(s) From 42caaf14de35e39ff2e843e9957c7a3f41702fa9 Mon Sep 17 00:00:00 2001 From: Bas van Doren Date: Sun, 6 Apr 2025 14:42:09 +0200 Subject: [PATCH 70/88] unix-ffi/machine: Use libc if librt is not present. Newer implementations of libc integrate the functions from librt, for example glibc since 2.17 and uClibc-ng. So if the librt.so cannot be loaded, it can be assumed that libc contains the expected functions. Signed-off-by: Bas van Doren --- unix-ffi/machine/machine/timer.py | 5 ++++- unix-ffi/machine/manifest.py | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/unix-ffi/machine/machine/timer.py b/unix-ffi/machine/machine/timer.py index 3f371142c..be00cee33 100644 --- a/unix-ffi/machine/machine/timer.py +++ b/unix-ffi/machine/machine/timer.py @@ -5,7 +5,10 @@ from signal import * libc = ffilib.libc() -librt = ffilib.open("librt") +try: + librt = ffilib.open("librt") +except OSError as e: + librt = libc CLOCK_REALTIME = 0 CLOCK_MONOTONIC = 1 diff --git a/unix-ffi/machine/manifest.py b/unix-ffi/machine/manifest.py index c0e40764d..f7c11b81a 100644 --- a/unix-ffi/machine/manifest.py +++ b/unix-ffi/machine/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.1") +metadata(version="0.2.2") # Originally written by Paul Sokolovsky. From 43ad7c58fd5fc247c255dacb37ab815ec212ee71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=BB=83=E6=98=95=E6=9A=90?= Date: Fri, 6 Dec 2024 13:43:24 +0800 Subject: [PATCH 71/88] requests: Use the host in the redirect url, not the one in headers. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The host in headers extracted from the original url may not be the same as the host in the redirect url. Poping out the host in headers force the code to use the host in the redirect url, otherwise the redirect may fail due to inconsistence of hosts in the original url and the redirect url. Signed-off-by: 黃昕暐 --- python-ecosys/requests/manifest.py | 2 +- python-ecosys/requests/requests/__init__.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py index f8343e2a1..85f159753 100644 --- a/python-ecosys/requests/manifest.py +++ b/python-ecosys/requests/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.10.1", pypi="requests") +metadata(version="0.10.2", pypi="requests") package("requests") diff --git a/python-ecosys/requests/requests/__init__.py b/python-ecosys/requests/requests/__init__.py index 2951035f7..4ca7489a4 100644 --- a/python-ecosys/requests/requests/__init__.py +++ b/python-ecosys/requests/requests/__init__.py @@ -182,6 +182,8 @@ def request( if redirect: s.close() + # Use the host specified in the redirect URL, as it may not be the same as the original URL. + headers.pop("Host", None) if status in [301, 302, 303]: return request("GET", redirect, None, None, headers, stream) else: From 86df7233019678616f864e4325460a2f893c5e7f Mon Sep 17 00:00:00 2001 From: FuNK3Y Date: Fri, 7 Feb 2025 19:53:53 +0100 Subject: [PATCH 72/88] aiohttp: Fix header case sensitivity. According to RFC https://datatracker.ietf.org/doc/html/rfc7230#section-3.2 header names are case-insensitive. This commit makes sure that the module behaves consistently regardless of the casing of "Content-type" and "Content-Length" (other headers are not considered by the module). Without this fix, the client seems to wait for the connection termination (~10 seconds) prior to returning any content if the casing of "Content-Length" is different. Signed-off-by: FuNK3Y --- python-ecosys/aiohttp/aiohttp/__init__.py | 12 +++++++++--- python-ecosys/aiohttp/manifest.py | 2 +- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/python-ecosys/aiohttp/aiohttp/__init__.py b/python-ecosys/aiohttp/aiohttp/__init__.py index 1565163c4..3f57bac83 100644 --- a/python-ecosys/aiohttp/aiohttp/__init__.py +++ b/python-ecosys/aiohttp/aiohttp/__init__.py @@ -18,8 +18,14 @@ class ClientResponse: def __init__(self, reader): self.content = reader + def _get_header(self, keyname, default): + for k in self.headers: + if k.lower() == keyname: + return self.headers[k] + return default + def _decode(self, data): - c_encoding = self.headers.get("Content-Encoding") + c_encoding = self._get_header("content-encoding", None) if c_encoding in ("gzip", "deflate", "gzip,deflate"): try: import deflate @@ -39,10 +45,10 @@ async def read(self, sz=-1): return self._decode(await self.content.read(sz)) async def text(self, encoding="utf-8"): - return (await self.read(int(self.headers.get("Content-Length", -1)))).decode(encoding) + return (await self.read(int(self._get_header("content-length", -1)))).decode(encoding) async def json(self): - return _json.loads(await self.read(int(self.headers.get("Content-Length", -1)))) + return _json.loads(await self.read(int(self._get_header("content-length", -1)))) def __repr__(self): return "" % (self.status, self.headers) diff --git a/python-ecosys/aiohttp/manifest.py b/python-ecosys/aiohttp/manifest.py index 748970e5b..5020ec527 100644 --- a/python-ecosys/aiohttp/manifest.py +++ b/python-ecosys/aiohttp/manifest.py @@ -1,6 +1,6 @@ metadata( description="HTTP client module for MicroPython asyncio module", - version="0.0.3", + version="0.0.4", pypi="aiohttp", ) From 05a56c3cad059245c62df5d76baa5ebc3340f812 Mon Sep 17 00:00:00 2001 From: jomas Date: Mon, 2 Dec 2024 11:23:07 +0100 Subject: [PATCH 73/88] aiohttp: Allow headers to be passed to a WebSocketClient. This commit will make it possible to add headers to a Websocket. Among other things, this allows making a connection to online MQTT brokers over websocket, using the header entry "Sec-WebSocket-Protocol":"mqtt" in the handshake of the upgrade protocol. Signed-off-by: Damien George --- python-ecosys/aiohttp/aiohttp/__init__.py | 2 +- python-ecosys/aiohttp/aiohttp/aiohttp_ws.py | 2 +- python-ecosys/aiohttp/manifest.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python-ecosys/aiohttp/aiohttp/__init__.py b/python-ecosys/aiohttp/aiohttp/__init__.py index 3f57bac83..8c5493f30 100644 --- a/python-ecosys/aiohttp/aiohttp/__init__.py +++ b/python-ecosys/aiohttp/aiohttp/__init__.py @@ -269,7 +269,7 @@ def ws_connect(self, url, ssl=None): return _WSRequestContextManager(self, self._ws_connect(url, ssl=ssl)) async def _ws_connect(self, url, ssl=None): - ws_client = WebSocketClient(None) + ws_client = WebSocketClient(self._base_headers.copy()) await ws_client.connect(url, ssl=ssl, handshake_request=self.request_raw) self._reader = ws_client.reader return ClientWebSocketResponse(ws_client) diff --git a/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py b/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py index 07d833730..6e0818c92 100644 --- a/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py +++ b/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py @@ -136,7 +136,7 @@ def _encode_websocket_frame(cls, opcode, payload): return frame + payload async def handshake(self, uri, ssl, req): - headers = {} + headers = self.params _http_proto = "http" if uri.protocol != "wss" else "https" url = f"{_http_proto}://{uri.hostname}:{uri.port}{uri.path or '/'}" key = binascii.b2a_base64(bytes(random.getrandbits(8) for _ in range(16)))[:-1] diff --git a/python-ecosys/aiohttp/manifest.py b/python-ecosys/aiohttp/manifest.py index 5020ec527..d22a6ce11 100644 --- a/python-ecosys/aiohttp/manifest.py +++ b/python-ecosys/aiohttp/manifest.py @@ -1,6 +1,6 @@ metadata( description="HTTP client module for MicroPython asyncio module", - version="0.0.4", + version="0.0.5", pypi="aiohttp", ) From 9307e21dfb34152ac605cd810447716b459d7f7d Mon Sep 17 00:00:00 2001 From: Matthias Urlichs Date: Wed, 26 Mar 2025 12:00:40 +0100 Subject: [PATCH 74/88] usb-device-cdc: Optimise writing small data so it doesn't require alloc. Only allocate a memoryview when the (first) write was partial. Signed-off-by: Matthias Urlichs --- micropython/usb/usb-device-cdc/manifest.py | 2 +- micropython/usb/usb-device-cdc/usb/device/cdc.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/micropython/usb/usb-device-cdc/manifest.py b/micropython/usb/usb-device-cdc/manifest.py index 4520325e3..e844b6f01 100644 --- a/micropython/usb/usb-device-cdc/manifest.py +++ b/micropython/usb/usb-device-cdc/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.1") +metadata(version="0.1.2") require("usb-device") package("usb") diff --git a/micropython/usb/usb-device-cdc/usb/device/cdc.py b/micropython/usb/usb-device-cdc/usb/device/cdc.py index 28bfb0657..0acea184f 100644 --- a/micropython/usb/usb-device-cdc/usb/device/cdc.py +++ b/micropython/usb/usb-device-cdc/usb/device/cdc.py @@ -350,10 +350,9 @@ def _rd_cb(self, ep, res, num_bytes): ### def write(self, buf): - # use a memoryview to track how much of 'buf' we've written so far - # (unfortunately, this means a 1 block allocation for each write, but it's otherwise allocation free.) start = time.ticks_ms() - mv = memoryview(buf) + mv = buf + while True: # Keep pushing buf into _wb into it's all gone nbytes = self._wb.write(mv) @@ -362,6 +361,10 @@ def write(self, buf): if nbytes == len(mv): return len(buf) # Success + # if buf couldn't be fully written on the first attempt + # convert it to a memoryview to track partial writes + if mv is buf: + mv = memoryview(buf) mv = mv[nbytes:] # check for timeout From 48bf3a74a8599aca7ddf8dfbb9d08d8cd5172d25 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 11 Apr 2025 12:23:33 +1000 Subject: [PATCH 75/88] inspect: Fix isgenerator logic. Also optimise both `isgenerator()` and `isgeneratorfunction()` so they use the same lambda, and don't have to create it each time they are called. Fixes issue #997. Signed-off-by: Damien George --- python-stdlib/inspect/inspect.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python-stdlib/inspect/inspect.py b/python-stdlib/inspect/inspect.py index 06aba8762..fb2ad885d 100644 --- a/python-stdlib/inspect/inspect.py +++ b/python-stdlib/inspect/inspect.py @@ -1,5 +1,7 @@ import sys +_g = lambda: (yield) + def getmembers(obj, pred=None): res = [] @@ -16,11 +18,11 @@ def isfunction(obj): def isgeneratorfunction(obj): - return isinstance(obj, type(lambda: (yield))) + return isinstance(obj, type(_g)) def isgenerator(obj): - return isinstance(obj, type(lambda: (yield)())) + return isinstance(obj, type((_g)())) class _Class: From 2665047fa7c88729e8d581bcf4ed047298c0dc30 Mon Sep 17 00:00:00 2001 From: Damien George Date: Fri, 11 Apr 2025 12:24:53 +1000 Subject: [PATCH 76/88] inspect: Add basic unit tests. Signed-off-by: Damien George --- python-stdlib/inspect/test_inspect.py | 54 +++++++++++++++++++++++++++ tools/ci.sh | 1 + 2 files changed, 55 insertions(+) create mode 100644 python-stdlib/inspect/test_inspect.py diff --git a/python-stdlib/inspect/test_inspect.py b/python-stdlib/inspect/test_inspect.py new file mode 100644 index 000000000..6f262ca64 --- /dev/null +++ b/python-stdlib/inspect/test_inspect.py @@ -0,0 +1,54 @@ +import inspect +import unittest + + +def fun(): + return 1 + + +def gen(): + yield 1 + + +class Class: + def meth(self): + pass + + +entities = ( + fun, + gen, + gen(), + Class, + Class.meth, + Class().meth, + inspect, +) + + +class TestInspect(unittest.TestCase): + def _test_is_helper(self, f, *entities_true): + for entity in entities: + result = f(entity) + if entity in entities_true: + self.assertTrue(result) + else: + self.assertFalse(result) + + def test_isfunction(self): + self._test_is_helper(inspect.isfunction, entities[0], entities[4]) + + def test_isgeneratorfunction(self): + self._test_is_helper(inspect.isgeneratorfunction, entities[1]) + + def test_isgenerator(self): + self._test_is_helper(inspect.isgenerator, entities[2]) + + def test_ismethod(self): + self._test_is_helper(inspect.ismethod, entities[5]) + + def test_isclass(self): + self._test_is_helper(inspect.isclass, entities[3]) + + def test_ismodule(self): + self._test_is_helper(inspect.ismodule, entities[6]) diff --git a/tools/ci.sh b/tools/ci.sh index a5fcdf22e..6689e8aa4 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -86,6 +86,7 @@ function ci_package_tests_run { python-stdlib/datetime \ python-stdlib/fnmatch \ python-stdlib/hashlib \ + python-stdlib/inspect \ python-stdlib/pathlib \ python-stdlib/quopri \ python-stdlib/shutil \ From 5b496e944ec045177afa1620920a168410b7f60b Mon Sep 17 00:00:00 2001 From: Damien George Date: Mon, 14 Apr 2025 10:24:54 +1000 Subject: [PATCH 77/88] inspect: Implement iscoroutinefunction and iscoroutine. Signed-off-by: Damien George --- python-stdlib/inspect/inspect.py | 5 +++++ python-stdlib/inspect/manifest.py | 2 +- python-stdlib/inspect/test_inspect.py | 6 ++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/python-stdlib/inspect/inspect.py b/python-stdlib/inspect/inspect.py index fb2ad885d..c16c6b3e3 100644 --- a/python-stdlib/inspect/inspect.py +++ b/python-stdlib/inspect/inspect.py @@ -25,6 +25,11 @@ def isgenerator(obj): return isinstance(obj, type((_g)())) +# In MicroPython there's currently no way to distinguish between generators and coroutines. +iscoroutinefunction = isgeneratorfunction +iscoroutine = isgenerator + + class _Class: def meth(): pass diff --git a/python-stdlib/inspect/manifest.py b/python-stdlib/inspect/manifest.py index a9d5a2381..e99e659f2 100644 --- a/python-stdlib/inspect/manifest.py +++ b/python-stdlib/inspect/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.2") +metadata(version="0.1.3") module("inspect.py") diff --git a/python-stdlib/inspect/test_inspect.py b/python-stdlib/inspect/test_inspect.py index 6f262ca64..29ed80f11 100644 --- a/python-stdlib/inspect/test_inspect.py +++ b/python-stdlib/inspect/test_inspect.py @@ -44,6 +44,12 @@ def test_isgeneratorfunction(self): def test_isgenerator(self): self._test_is_helper(inspect.isgenerator, entities[2]) + def test_iscoroutinefunction(self): + self._test_is_helper(inspect.iscoroutinefunction, entities[1]) + + def test_iscoroutine(self): + self._test_is_helper(inspect.iscoroutine, entities[2]) + def test_ismethod(self): self._test_is_helper(inspect.ismethod, entities[5]) From f8c8875e250e1dfef0158f15ccbb66e8cee9aa57 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Thu, 24 Apr 2025 10:34:45 +1000 Subject: [PATCH 78/88] lora: Fix SNR value in SX126x received packets. Wasn't being treated as a signed value. Fixes issue #999. Signed-off-by: Angus Gratton --- micropython/lora/lora-sx126x/lora/sx126x.py | 5 +++-- micropython/lora/lora-sx126x/manifest.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/micropython/lora/lora-sx126x/lora/sx126x.py b/micropython/lora/lora-sx126x/lora/sx126x.py index 641367a9f..7fa4896ae 100644 --- a/micropython/lora/lora-sx126x/lora/sx126x.py +++ b/micropython/lora/lora-sx126x/lora/sx126x.py @@ -596,8 +596,9 @@ def _read_packet(self, rx_packet, flags): pkt_status = self._cmd("B", _CMD_GET_PACKET_STATUS, n_read=4) rx_packet.ticks_ms = ticks_ms - rx_packet.snr = pkt_status[2] # SNR, units: dB *4 - rx_packet.rssi = 0 - pkt_status[1] // 2 # RSSI, units: dBm + # SNR units are dB * 4 (signed) + rx_packet.rssi, rx_packet.snr = struct.unpack("xBbx", pkt_status) + rx_packet.rssi //= -2 # RSSI, units: dBm rx_packet.crc_error = (flags & _IRQ_CRC_ERR) != 0 return rx_packet diff --git a/micropython/lora/lora-sx126x/manifest.py b/micropython/lora/lora-sx126x/manifest.py index 038710820..76fa91d8d 100644 --- a/micropython/lora/lora-sx126x/manifest.py +++ b/micropython/lora/lora-sx126x/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.4") +metadata(version="0.1.5") require("lora") package("lora") From d887a021e831ee3b7e6f00f6a4c32b36ee6c4769 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Thu, 24 Apr 2025 10:41:10 +1000 Subject: [PATCH 79/88] top: Bump the Ruff version to 0.11.6. With small code fixes to match. Signed-off-by: Angus Gratton --- .github/workflows/ruff.yml | 3 ++- .pre-commit-config.yaml | 3 ++- .../bluetooth/aioble/examples/l2cap_file_client.py | 2 +- .../bluetooth/aioble/examples/l2cap_file_server.py | 2 +- micropython/drivers/codec/wm8960/wm8960.py | 12 ++++-------- micropython/drivers/display/lcd160cr/lcd160cr.py | 6 ++---- pyproject.toml | 3 ++- 7 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 71c4131f0..b347e34ee 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -6,6 +6,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: pip install --user ruff==0.1.2 + # Version should be kept in sync with .pre-commit_config.yaml & also micropython + - run: pip install --user ruff==0.11.6 - run: ruff check --output-format=github . - run: ruff format --diff . diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 335c1c2fc..05f5d3df0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,8 @@ repos: verbose: true stages: [commit-msg] - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: v0.1.2 + # Version should be kept in sync with .github/workflows/ruff.yml & also micropython + rev: v0.11.6 hooks: - id: ruff id: ruff-format diff --git a/micropython/bluetooth/aioble/examples/l2cap_file_client.py b/micropython/bluetooth/aioble/examples/l2cap_file_client.py index 9dce349a7..0817ca162 100644 --- a/micropython/bluetooth/aioble/examples/l2cap_file_client.py +++ b/micropython/bluetooth/aioble/examples/l2cap_file_client.py @@ -88,7 +88,7 @@ async def download(self, path, dest): await self._command(_COMMAND_SEND, path.encode()) - with open(dest, "wb") as f: # noqa: ASYNC101 + with open(dest, "wb") as f: # noqa: ASYNC230 total = 0 buf = bytearray(self._channel.our_mtu) mv = memoryview(buf) diff --git a/micropython/bluetooth/aioble/examples/l2cap_file_server.py b/micropython/bluetooth/aioble/examples/l2cap_file_server.py index fb806effc..3c3b3b44d 100644 --- a/micropython/bluetooth/aioble/examples/l2cap_file_server.py +++ b/micropython/bluetooth/aioble/examples/l2cap_file_server.py @@ -83,7 +83,7 @@ async def l2cap_task(connection): if send_file: print("Sending:", send_file) - with open(send_file, "rb") as f: # noqa: ASYNC101 + with open(send_file, "rb") as f: # noqa: ASYNC230 buf = bytearray(channel.peer_mtu) mv = memoryview(buf) while n := f.readinto(buf): diff --git a/micropython/drivers/codec/wm8960/wm8960.py b/micropython/drivers/codec/wm8960/wm8960.py index dc0dd655d..313649f36 100644 --- a/micropython/drivers/codec/wm8960/wm8960.py +++ b/micropython/drivers/codec/wm8960/wm8960.py @@ -331,8 +331,7 @@ def __init__( sysclk = 11289600 else: sysclk = 12288000 - if sysclk < sample_rate * 256: - sysclk = sample_rate * 256 + sysclk = max(sysclk, sample_rate * 256) if mclk_freq is None: mclk_freq = sysclk else: # sysclk_source == SYSCLK_MCLK @@ -691,10 +690,8 @@ def alc_mode(self, channel, mode=ALC_MODE): def alc_gain(self, target=-12, max_gain=30, min_gain=-17.25, noise_gate=-78): def limit(value, minval, maxval): value = int(value) - if value < minval: - value = minval - if value > maxval: - value = maxval + value = max(value, minval) + value = min(value, maxval) return value target = limit((16 + (target * 2) // 3), 0, 15) @@ -718,8 +715,7 @@ def logb(value, limit): while value > 1: value >>= 1 lb += 1 - if lb > limit: - lb = limit + lb = min(lb, limit) return lb attack = logb(attack / 6, 7) diff --git a/micropython/drivers/display/lcd160cr/lcd160cr.py b/micropython/drivers/display/lcd160cr/lcd160cr.py index 42b5e215b..177c6fea3 100644 --- a/micropython/drivers/display/lcd160cr/lcd160cr.py +++ b/micropython/drivers/display/lcd160cr/lcd160cr.py @@ -189,10 +189,8 @@ def clip_line(c, w, h): c[3] = h - 1 else: if c[0] == c[2]: - if c[1] < 0: - c[1] = 0 - if c[3] < 0: - c[3] = 0 + c[1] = max(c[1], 0) + c[3] = max(c[3], 0) else: if c[3] < c[1]: c[0], c[2] = c[2], c[0] diff --git a/pyproject.toml b/pyproject.toml index 4776ddfe9..83d29405d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ ignore = [ "ISC003", # micropython does not support implicit concatenation of f-strings "PIE810", # micropython does not support passing tuples to .startswith or .endswith "PLC1901", - "PLR1701", + "PLR1704", # sometimes desirable to redefine an argument to save code size "PLR1714", "PLR5501", "PLW0602", @@ -72,6 +72,7 @@ ignore = [ "PLW2901", "RUF012", "RUF100", + "SIM101", "W191", # tab-indent, redundant when using formatter ] line-length = 99 From 68e0dfce0a8708e7d9aef4446fad842cc5929410 Mon Sep 17 00:00:00 2001 From: Angus Gratton Date: Thu, 24 Apr 2025 11:01:42 +1000 Subject: [PATCH 80/88] all: Apply Ruff 0.11.6 reformatting changes. Signed-off-by: Angus Gratton --- micropython/aiorepl/aiorepl.py | 16 ++++++++-------- micropython/drivers/imu/lsm9ds1/lsm9ds1.py | 1 + micropython/drivers/radio/nrf24l01/nrf24l01.py | 3 +-- micropython/drivers/sensor/hts221/hts221.py | 2 +- micropython/drivers/sensor/lps22h/lps22h.py | 1 + micropython/espflash/espflash.py | 16 ++++++++-------- .../lora/examples/reliable_delivery/sender.py | 2 +- .../examples/reliable_delivery/sender_async.py | 2 +- micropython/senml/examples/actuator.py | 1 - micropython/senml/examples/base.py | 1 - micropython/senml/examples/basic.py | 1 - micropython/senml/examples/basic2.py | 1 - micropython/senml/examples/basic_cbor.py | 1 - micropython/senml/examples/custom_record.py | 1 - micropython/senml/examples/gateway.py | 1 - micropython/senml/examples/gateway_actuators.py | 1 - .../senml/examples/supported_data_types.py | 1 - micropython/senml/senml/__init__.py | 1 - micropython/senml/senml/senml_pack.py | 1 - micropython/senml/senml/senml_record.py | 1 - python-ecosys/cbor2/cbor2/__init__.py | 1 - python-ecosys/cbor2/cbor2/_decoder.py | 1 - python-ecosys/cbor2/cbor2/_encoder.py | 1 - python-ecosys/cbor2/examples/cbor_test.py | 1 - 24 files changed, 22 insertions(+), 37 deletions(-) diff --git a/micropython/aiorepl/aiorepl.py b/micropython/aiorepl/aiorepl.py index 8f45dfac0..3f437459d 100644 --- a/micropython/aiorepl/aiorepl.py +++ b/micropython/aiorepl/aiorepl.py @@ -132,7 +132,7 @@ async def task(g=None, prompt="--> "): continue if curs: # move cursor to end of the line - sys.stdout.write("\x1B[{}C".format(curs)) + sys.stdout.write("\x1b[{}C".format(curs)) curs = 0 sys.stdout.write("\n") if cmd: @@ -153,10 +153,10 @@ async def task(g=None, prompt="--> "): if curs: cmd = "".join((cmd[: -curs - 1], cmd[-curs:])) sys.stdout.write( - "\x08\x1B[K" + "\x08\x1b[K" ) # move cursor back, erase to end of line sys.stdout.write(cmd[-curs:]) # redraw line - sys.stdout.write("\x1B[{}D".format(curs)) # reset cursor location + sys.stdout.write("\x1b[{}D".format(curs)) # reset cursor location else: cmd = cmd[:-1] sys.stdout.write("\x08 \x08") @@ -207,21 +207,21 @@ async def task(g=None, prompt="--> "): elif key == "[D": # left if curs < len(cmd) - 1: curs += 1 - sys.stdout.write("\x1B") + sys.stdout.write("\x1b") sys.stdout.write(key) elif key == "[C": # right if curs: curs -= 1 - sys.stdout.write("\x1B") + sys.stdout.write("\x1b") sys.stdout.write(key) elif key == "[H": # home pcurs = curs curs = len(cmd) - sys.stdout.write("\x1B[{}D".format(curs - pcurs)) # move cursor left + sys.stdout.write("\x1b[{}D".format(curs - pcurs)) # move cursor left elif key == "[F": # end pcurs = curs curs = 0 - sys.stdout.write("\x1B[{}C".format(pcurs)) # move cursor right + sys.stdout.write("\x1b[{}C".format(pcurs)) # move cursor right else: # sys.stdout.write("\\x") # sys.stdout.write(hex(c)) @@ -231,7 +231,7 @@ async def task(g=None, prompt="--> "): # inserting into middle of line cmd = "".join((cmd[:-curs], b, cmd[-curs:])) sys.stdout.write(cmd[-curs - 1 :]) # redraw line to end - sys.stdout.write("\x1B[{}D".format(curs)) # reset cursor location + sys.stdout.write("\x1b[{}D".format(curs)) # reset cursor location else: sys.stdout.write(b) cmd += b diff --git a/micropython/drivers/imu/lsm9ds1/lsm9ds1.py b/micropython/drivers/imu/lsm9ds1/lsm9ds1.py index e3d46429d..e5a96ad5c 100644 --- a/micropython/drivers/imu/lsm9ds1/lsm9ds1.py +++ b/micropython/drivers/imu/lsm9ds1/lsm9ds1.py @@ -43,6 +43,7 @@ print("") time.sleep_ms(100) """ + import array from micropython import const diff --git a/micropython/drivers/radio/nrf24l01/nrf24l01.py b/micropython/drivers/radio/nrf24l01/nrf24l01.py index d015250cf..9fbadcd60 100644 --- a/micropython/drivers/radio/nrf24l01/nrf24l01.py +++ b/micropython/drivers/radio/nrf24l01/nrf24l01.py @@ -1,5 +1,4 @@ -"""NRF24L01 driver for MicroPython -""" +"""NRF24L01 driver for MicroPython""" from micropython import const import utime diff --git a/micropython/drivers/sensor/hts221/hts221.py b/micropython/drivers/sensor/hts221/hts221.py index fec52a738..c6cd51f48 100644 --- a/micropython/drivers/sensor/hts221/hts221.py +++ b/micropython/drivers/sensor/hts221/hts221.py @@ -52,7 +52,7 @@ def __init__(self, i2c, data_rate=1, address=0x5F): # Set configuration register # Humidity and temperature average configuration - self.bus.writeto_mem(self.slv_addr, 0x10, b"\x1B") + self.bus.writeto_mem(self.slv_addr, 0x10, b"\x1b") # Set control register # PD | BDU | ODR diff --git a/micropython/drivers/sensor/lps22h/lps22h.py b/micropython/drivers/sensor/lps22h/lps22h.py index 1e7f4ec3e..7dec72528 100644 --- a/micropython/drivers/sensor/lps22h/lps22h.py +++ b/micropython/drivers/sensor/lps22h/lps22h.py @@ -37,6 +37,7 @@ print("Pressure: %.2f hPa Temperature: %.2f C"%(lps.pressure(), lps.temperature())) time.sleep_ms(10) """ + import machine from micropython import const diff --git a/micropython/espflash/espflash.py b/micropython/espflash/espflash.py index 74988777a..fbf4e1f7e 100644 --- a/micropython/espflash/espflash.py +++ b/micropython/espflash/espflash.py @@ -113,22 +113,22 @@ def _poll_reg(self, addr, flag, retry=10, delay=0.050): raise Exception(f"Register poll timeout. Addr: 0x{addr:02X} Flag: 0x{flag:02X}.") def _write_slip(self, pkt): - pkt = pkt.replace(b"\xDB", b"\xdb\xdd").replace(b"\xc0", b"\xdb\xdc") - self.uart.write(b"\xC0" + pkt + b"\xC0") + pkt = pkt.replace(b"\xdb", b"\xdb\xdd").replace(b"\xc0", b"\xdb\xdc") + self.uart.write(b"\xc0" + pkt + b"\xc0") self._log(pkt) def _read_slip(self): pkt = None # Find the packet start. - if self.uart.read(1) == b"\xC0": + if self.uart.read(1) == b"\xc0": pkt = bytearray() while True: b = self.uart.read(1) - if b is None or b == b"\xC0": + if b is None or b == b"\xc0": break pkt += b - pkt = pkt.replace(b"\xDB\xDD", b"\xDB").replace(b"\xDB\xDC", b"\xC0") - self._log(b"\xC0" + pkt + b"\xC0", False) + pkt = pkt.replace(b"\xdb\xdd", b"\xdb").replace(b"\xdb\xdc", b"\xc0") + self._log(b"\xc0" + pkt + b"\xc0", False) return pkt def _strerror(self, err): @@ -230,7 +230,7 @@ def flash_read_size(self): raise Exception(f"Unexpected flash size bits: 0x{flash_bits:02X}.") flash_size = 2**flash_bits - print(f"Flash size {flash_size/1024/1024} MBytes") + print(f"Flash size {flash_size / 1024 / 1024} MBytes") return flash_size def flash_attach(self): @@ -265,7 +265,7 @@ def flash_write_file(self, path, blksize=0x1000): self.md5sum.update(buf) # The last data block should be padded to the block size with 0xFF bytes. if len(buf) < blksize: - buf += b"\xFF" * (blksize - len(buf)) + buf += b"\xff" * (blksize - len(buf)) checksum = self._checksum(buf) if seq % erase_blocks == 0: # print(f"Erasing {seq} -> {seq+erase_blocks}...") diff --git a/micropython/lora/examples/reliable_delivery/sender.py b/micropython/lora/examples/reliable_delivery/sender.py index 2fba0d4d7..957e9d824 100644 --- a/micropython/lora/examples/reliable_delivery/sender.py +++ b/micropython/lora/examples/reliable_delivery/sender.py @@ -149,7 +149,7 @@ def send(self, sensor_data, adjust_output_power=True): delta = time.ticks_diff(maybe_ack.ticks_ms, sent_at) print( f"ACKed with RSSI {rssi}, {delta}ms after sent " - + f"(skew {delta-ACK_DELAY_MS-ack_packet_ms}ms)" + + f"(skew {delta - ACK_DELAY_MS - ack_packet_ms}ms)" ) if adjust_output_power: diff --git a/micropython/lora/examples/reliable_delivery/sender_async.py b/micropython/lora/examples/reliable_delivery/sender_async.py index a27420f6b..4c14d6f11 100644 --- a/micropython/lora/examples/reliable_delivery/sender_async.py +++ b/micropython/lora/examples/reliable_delivery/sender_async.py @@ -141,7 +141,7 @@ async def send(self, sensor_data, adjust_output_power=True): delta = time.ticks_diff(maybe_ack.ticks_ms, sent_at) print( f"ACKed with RSSI {rssi}, {delta}ms after sent " - + f"(skew {delta-ACK_DELAY_MS-ack_packet_ms}ms)" + + f"(skew {delta - ACK_DELAY_MS - ack_packet_ms}ms)" ) if adjust_output_power: diff --git a/micropython/senml/examples/actuator.py b/micropython/senml/examples/actuator.py index 8e254349d..2fac474cd 100644 --- a/micropython/senml/examples/actuator.py +++ b/micropython/senml/examples/actuator.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from senml import * diff --git a/micropython/senml/examples/base.py b/micropython/senml/examples/base.py index 426cbbd0e..6a49cfdd2 100644 --- a/micropython/senml/examples/base.py +++ b/micropython/senml/examples/base.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from senml import * import time diff --git a/micropython/senml/examples/basic.py b/micropython/senml/examples/basic.py index 18a3a9a06..3f3ed6150 100644 --- a/micropython/senml/examples/basic.py +++ b/micropython/senml/examples/basic.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from senml import * import time diff --git a/micropython/senml/examples/basic2.py b/micropython/senml/examples/basic2.py index c2ea153bd..ca53b4a6e 100644 --- a/micropython/senml/examples/basic2.py +++ b/micropython/senml/examples/basic2.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from senml import * import time diff --git a/micropython/senml/examples/basic_cbor.py b/micropython/senml/examples/basic_cbor.py index 804a886fc..b9d9d620b 100644 --- a/micropython/senml/examples/basic_cbor.py +++ b/micropython/senml/examples/basic_cbor.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from senml import * import time import cbor2 diff --git a/micropython/senml/examples/custom_record.py b/micropython/senml/examples/custom_record.py index e68d05f5b..1e83ea06b 100644 --- a/micropython/senml/examples/custom_record.py +++ b/micropython/senml/examples/custom_record.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from senml import * import time diff --git a/micropython/senml/examples/gateway.py b/micropython/senml/examples/gateway.py index d28e4cffc..e1827ff2d 100644 --- a/micropython/senml/examples/gateway.py +++ b/micropython/senml/examples/gateway.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from senml import * import time diff --git a/micropython/senml/examples/gateway_actuators.py b/micropython/senml/examples/gateway_actuators.py index add5ed24c..a7e5b378c 100644 --- a/micropython/senml/examples/gateway_actuators.py +++ b/micropython/senml/examples/gateway_actuators.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from senml import * diff --git a/micropython/senml/examples/supported_data_types.py b/micropython/senml/examples/supported_data_types.py index 3149f49d2..94976bb66 100644 --- a/micropython/senml/examples/supported_data_types.py +++ b/micropython/senml/examples/supported_data_types.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from senml import * import time diff --git a/micropython/senml/senml/__init__.py b/micropython/senml/senml/__init__.py index 93cbd7700..908375fdb 100644 --- a/micropython/senml/senml/__init__.py +++ b/micropython/senml/senml/__init__.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from .senml_base import SenmlBase from .senml_pack import SenmlPack from .senml_record import SenmlRecord diff --git a/micropython/senml/senml/senml_pack.py b/micropython/senml/senml/senml_pack.py index 4e106fd3e..5a0554467 100644 --- a/micropython/senml/senml/senml_pack.py +++ b/micropython/senml/senml/senml_pack.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from senml.senml_record import SenmlRecord from senml.senml_base import SenmlBase import json diff --git a/micropython/senml/senml/senml_record.py b/micropython/senml/senml/senml_record.py index 9dfe22873..ae40f0f70 100644 --- a/micropython/senml/senml/senml_record.py +++ b/micropython/senml/senml/senml_record.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - import binascii from senml.senml_base import SenmlBase diff --git a/python-ecosys/cbor2/cbor2/__init__.py b/python-ecosys/cbor2/cbor2/__init__.py index 7cd98734e..80790f0da 100644 --- a/python-ecosys/cbor2/cbor2/__init__.py +++ b/python-ecosys/cbor2/cbor2/__init__.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - from ._decoder import CBORDecoder from ._decoder import load from ._decoder import loads diff --git a/python-ecosys/cbor2/cbor2/_decoder.py b/python-ecosys/cbor2/cbor2/_decoder.py index 5d509a535..965dbfd46 100644 --- a/python-ecosys/cbor2/cbor2/_decoder.py +++ b/python-ecosys/cbor2/cbor2/_decoder.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - import io import struct diff --git a/python-ecosys/cbor2/cbor2/_encoder.py b/python-ecosys/cbor2/cbor2/_encoder.py index 80a4ac022..fe8715468 100644 --- a/python-ecosys/cbor2/cbor2/_encoder.py +++ b/python-ecosys/cbor2/cbor2/_encoder.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - import io import struct diff --git a/python-ecosys/cbor2/examples/cbor_test.py b/python-ecosys/cbor2/examples/cbor_test.py index b4f351786..a1cd7e93e 100644 --- a/python-ecosys/cbor2/examples/cbor_test.py +++ b/python-ecosys/cbor2/examples/cbor_test.py @@ -23,7 +23,6 @@ THE SOFTWARE. """ - import cbor2 input = [ From 0d60de65b7cf6bf784e2e4e9f84f1a70faf2ae83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20de=20Giessen?= Date: Thu, 7 Mar 2024 15:01:10 +0100 Subject: [PATCH 81/88] utop: Add initial implementation for ESP32. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniël van de Giessen --- micropython/utop/README.md | 12 +++++ micropython/utop/manifest.py | 6 +++ micropython/utop/utop.py | 85 ++++++++++++++++++++++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 micropython/utop/README.md create mode 100644 micropython/utop/manifest.py create mode 100644 micropython/utop/utop.py diff --git a/micropython/utop/README.md b/micropython/utop/README.md new file mode 100644 index 000000000..7b07e785d --- /dev/null +++ b/micropython/utop/README.md @@ -0,0 +1,12 @@ +# utop + +Provides a top-like live overview of the running system. + +On the `esp32` port this depends on the `esp32.idf_task_info()` function, which +can be enabled by adding the following lines to the board `sdkconfig`: + +```ini +CONFIG_FREERTOS_USE_TRACE_FACILITY=y +CONFIG_FREERTOS_VTASKLIST_INCLUDE_COREID=y +CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=y +``` diff --git a/micropython/utop/manifest.py b/micropython/utop/manifest.py new file mode 100644 index 000000000..ebba07270 --- /dev/null +++ b/micropython/utop/manifest.py @@ -0,0 +1,6 @@ +metadata( + version="0.1.0", + description="Provides a top-like live overview of the running system.", +) + +module("utop.py") diff --git a/micropython/utop/utop.py b/micropython/utop/utop.py new file mode 100644 index 000000000..d4e94c94a --- /dev/null +++ b/micropython/utop/utop.py @@ -0,0 +1,85 @@ +import time + +try: + import esp32 + import _thread +except ImportError: + esp32 = None + + +def top(update_interval_ms=1000, timeout_ms=None, thread_names={}): + time_start = time.ticks_ms() + previous_total_runtime = None + previous_task_runtimes = {} + previous_line_count = 0 + esp32_task_state_names = dict( + enumerate(("running", "ready", "blocked", "suspended", "deleted", "invalid")) + ) + + while timeout_ms is None or abs(time.ticks_diff(time.ticks_ms(), time_start)) < timeout_ms: + if previous_line_count > 0: + print("\x1b[{}A".format(previous_line_count), end="") + line_count = 0 + + if esp32 is not None: + if not hasattr(esp32, "idf_task_info"): + print( + "INFO: esp32.idf_task_info() is not available, cannot list active tasks.\x1b[K" + ) + line_count += 1 + else: + print(" CPU% CORE PRIORITY STATE STACKWATERMARK NAME\x1b[K") + line_count += 1 + + total_runtime, tasks = esp32.idf_task_info() + current_thread_id = _thread.get_ident() + tasks.sort(key=lambda t: t[0]) + for ( + task_id, + task_name, + task_state, + task_priority, + task_runtime, + task_stackhighwatermark, + task_coreid, + ) in tasks: + task_runtime_percentage = "-" + if ( + total_runtime != previous_total_runtime + and task_id in previous_task_runtimes + ): + task_runtime_percentage = "{:.2f}%".format( + 100 + * ((task_runtime - previous_task_runtimes[task_id]) & (2**32 - 1)) + / ((total_runtime - previous_total_runtime) & (2**32 - 1)) + ) + print( + "{:>7} {:>4} {:>8d} {:<9} {:<14d} {}{}\x1b[K".format( + task_runtime_percentage, + "-" if task_coreid is None else task_coreid, + task_priority, + esp32_task_state_names.get(task_state, "unknown"), + task_stackhighwatermark, + thread_names.get(task_id, task_name), + " (*)" if task_id == current_thread_id else "", + ) + ) + line_count += 1 + + previous_task_runtimes[task_id] = task_runtime + previous_total_runtime = total_runtime + else: + print("INFO: Platform does not support listing active tasks.\x1b[K") + line_count += 1 + + if previous_line_count > line_count: + for _ in range(previous_line_count - line_count): + print("\x1b[K") + print("\x1b[{}A".format(previous_line_count - line_count), end="") + + previous_line_count = line_count + + try: + time.sleep_ms(update_interval_ms) + except KeyboardInterrupt: + break From 08e09afbe3d16ea97b78ba486153a7ea4a750232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20de=20Giessen?= Date: Mon, 12 May 2025 16:24:19 +0200 Subject: [PATCH 82/88] utop: Print MicroPython memory info. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniël van de Giessen --- micropython/utop/utop.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/micropython/utop/utop.py b/micropython/utop/utop.py index d4e94c94a..76fdaade0 100644 --- a/micropython/utop/utop.py +++ b/micropython/utop/utop.py @@ -1,3 +1,4 @@ +import micropython import time try: @@ -72,6 +73,12 @@ def top(update_interval_ms=1000, timeout_ms=None, thread_names={}): print("INFO: Platform does not support listing active tasks.\x1b[K") line_count += 1 + print("\x1b[K") + line_count += 1 + print("MicroPython ", end="") + micropython.mem_info() + line_count += 3 + if previous_line_count > line_count: for _ in range(previous_line_count - line_count): print("\x1b[K") From 54d5f7cee2b95c976589bd4c815c24c358557e3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20de=20Giessen?= Date: Thu, 7 Mar 2024 15:01:45 +0100 Subject: [PATCH 83/88] utop: Print IDF heap details. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniël van de Giessen --- micropython/utop/utop.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/micropython/utop/utop.py b/micropython/utop/utop.py index 76fdaade0..799ff4212 100644 --- a/micropython/utop/utop.py +++ b/micropython/utop/utop.py @@ -79,6 +79,23 @@ def top(update_interval_ms=1000, timeout_ms=None, thread_names={}): micropython.mem_info() line_count += 3 + if esp32 is not None: + print("\x1b[K") + line_count += 1 + for name, cap in (("data", esp32.HEAP_DATA), ("exec", esp32.HEAP_EXEC)): + heaps = esp32.idf_heap_info(cap) + print( + "IDF heap ({}): {} regions, {} total, {} free, {} largest contiguous, {} min free watermark\x1b[K".format( + name, + len(heaps), + sum((h[0] for h in heaps)), + sum((h[1] for h in heaps)), + max((h[2] for h in heaps)), + sum((h[3] for h in heaps)), + ) + ) + line_count += 1 + if previous_line_count > line_count: for _ in range(previous_line_count - line_count): print("\x1b[K") From 567540d4e0be3fd4500cc199416bfa602bf7bcec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20de=20Giessen?= Date: Wed, 14 May 2025 18:08:22 +0200 Subject: [PATCH 84/88] tools/verifygitlog.py: Sync with changes from the main repo. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This includes the following commits: - Allow long co-author and sign-off names. - Disallow a leading slash in commit subject line. - Apply stricter rules on git subject line. - Show invalid commit subjects in quotes. Signed-off-by: Daniël van de Giessen --- tools/verifygitlog.py | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/tools/verifygitlog.py b/tools/verifygitlog.py index 20be794f8..46fec1e0c 100755 --- a/tools/verifygitlog.py +++ b/tools/verifygitlog.py @@ -49,7 +49,7 @@ def git_log(pretty_format, *args): def diagnose_subject_line(subject_line, subject_line_format, err): - err.error("Subject line: " + subject_line) + err.error('Subject line: "' + subject_line + '"') if not subject_line.endswith("."): err.error('* must end with "."') if not re.match(r"^[^!]+: ", subject_line): @@ -98,20 +98,47 @@ def verify_message_body(raw_body, err): if len(subject_line) >= 73: err.error("Subject line must be 72 or fewer characters: " + subject_line) + # Do additional checks on the prefix of the subject line. + verify_subject_line_prefix(subject_line.split(": ")[0], err) + # Second one divides subject and body. if len(raw_body) > 1 and raw_body[1]: err.error("Second message line must be empty: " + raw_body[1]) # Message body lines. for line in raw_body[2:]: - # Long lines with URLs are exempt from the line length rule. - if len(line) >= 76 and "://" not in line: + # Long lines with URLs or human names are exempt from the line length rule. + if len(line) >= 76 and not ( + "://" in line + or line.startswith("Co-authored-by: ") + or line.startswith("Signed-off-by: ") + ): err.error("Message lines should be 75 or less characters: " + line) if not raw_body[-1].startswith("Signed-off-by: ") or "@" not in raw_body[-1]: err.error('Message must be signed-off. Use "git commit -s".') +def verify_subject_line_prefix(prefix, err): + ext = (".c", ".h", ".cpp", ".js", ".rst", ".md") + + if prefix.startswith((".", "/")): + err.error('Subject prefix cannot begin with "." or "/".') + + if prefix.endswith("/"): + err.error('Subject prefix cannot end with "/".') + + if prefix.startswith("ports/"): + err.error( + 'Subject prefix cannot begin with "ports/", start with the name of the port instead.' + ) + + if prefix.endswith(ext): + err.error( + "Subject prefix cannot end with a file extension, use the main part of the filename without the extension." + ) + + def run(args): verbose("run", *args) From 96bd01ec047923c15bf936d2f77043968745542d Mon Sep 17 00:00:00 2001 From: Adam Lewis Date: Sun, 22 Sep 2024 17:55:35 +0200 Subject: [PATCH 85/88] urllib.urequest: Add support for headers to urequest.urlopen. This is an extension to CPython, similar to the `method` argument. Signed-off-by: Adam Lewis --- micropython/urllib.urequest/manifest.py | 2 +- micropython/urllib.urequest/urllib/urequest.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/micropython/urllib.urequest/manifest.py b/micropython/urllib.urequest/manifest.py index 2790208a7..033022711 100644 --- a/micropython/urllib.urequest/manifest.py +++ b/micropython/urllib.urequest/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.7.0") +metadata(version="0.8.0") # Originally written by Paul Sokolovsky. diff --git a/micropython/urllib.urequest/urllib/urequest.py b/micropython/urllib.urequest/urllib/urequest.py index f83cbaa94..a9622ea89 100644 --- a/micropython/urllib.urequest/urllib/urequest.py +++ b/micropython/urllib.urequest/urllib/urequest.py @@ -1,7 +1,7 @@ import socket -def urlopen(url, data=None, method="GET"): +def urlopen(url, data=None, method="GET", headers={}): if data is not None and method == "GET": method = "POST" try: @@ -40,6 +40,12 @@ def urlopen(url, data=None, method="GET"): s.write(host) s.write(b"\r\n") + for k in headers: + s.write(k) + s.write(b": ") + s.write(headers[k]) + s.write(b"\r\n") + if data: s.write(b"Content-Length: ") s.write(str(len(data))) From aad2e4809868354155c82de673c93abf0a38a194 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 5 May 2025 21:03:14 +1000 Subject: [PATCH 86/88] aiorepl: Use blocking reads for raw REPL and raw paste. Raw REPL mode is generally used as a command channel where all stdio traffic is related directly to the raw commands and responses sent. For this to work in aiorepl we need to ensure background tasks don't sent/ receive anything on stdio else the command channel will be corrupted. The simplest way to achieve this is to make the raw commands blocking and atomic rather than asyncio, assuming the user wont leave the device in raw mode for too long at any one time. Signed-off-by: Andrew Leech --- micropython/aiorepl/aiorepl.py | 17 +++++++++++------ micropython/aiorepl/manifest.py | 2 +- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/micropython/aiorepl/aiorepl.py b/micropython/aiorepl/aiorepl.py index 3f437459d..a640efcd1 100644 --- a/micropython/aiorepl/aiorepl.py +++ b/micropython/aiorepl/aiorepl.py @@ -161,7 +161,7 @@ async def task(g=None, prompt="--> "): cmd = cmd[:-1] sys.stdout.write("\x08 \x08") elif c == CHAR_CTRL_A: - await raw_repl(s, g) + raw_repl(sys.stdin, g) break elif c == CHAR_CTRL_B: continue @@ -239,7 +239,7 @@ async def task(g=None, prompt="--> "): micropython.kbd_intr(3) -async def raw_paste(s, g, window=512): +def raw_paste(s, window=512): sys.stdout.write("R\x01") # supported sys.stdout.write(bytearray([window & 0xFF, window >> 8, 0x01]).decode()) eof = False @@ -248,7 +248,7 @@ async def raw_paste(s, g, window=512): file = b"" while not eof: for idx in range(window): - b = await s.read(1) + b = s.read(1) c = ord(b) if c == CHAR_CTRL_C or c == CHAR_CTRL_D: # end of file @@ -267,7 +267,12 @@ async def raw_paste(s, g, window=512): return file -async def raw_repl(s: asyncio.StreamReader, g: dict): +def raw_repl(s, g: dict): + """ + This function is blocking to prevent other + async tasks from writing to the stdio stream and + breaking the raw repl session. + """ heading = "raw REPL; CTRL-B to exit\n" line = "" sys.stdout.write(heading) @@ -276,7 +281,7 @@ async def raw_repl(s: asyncio.StreamReader, g: dict): line = "" sys.stdout.write(">") while True: - b = await s.read(1) + b = s.read(1) c = ord(b) if c == CHAR_CTRL_A: rline = line @@ -284,7 +289,7 @@ async def raw_repl(s: asyncio.StreamReader, g: dict): if len(rline) == 2 and ord(rline[0]) == CHAR_CTRL_E: if rline[1] == "A": - line = await raw_paste(s, g) + line = raw_paste(s) break else: # reset raw REPL diff --git a/micropython/aiorepl/manifest.py b/micropython/aiorepl/manifest.py index 0fcc21849..ca2fa1513 100644 --- a/micropython/aiorepl/manifest.py +++ b/micropython/aiorepl/manifest.py @@ -1,5 +1,5 @@ metadata( - version="0.2.0", + version="0.2.1", description="Provides an asynchronous REPL that can run concurrently with an asyncio, also allowing await expressions.", ) From 15a6233d040d18bf151c93675c9c4da793d5220e Mon Sep 17 00:00:00 2001 From: Nick Budak Date: Wed, 4 Jun 2025 09:55:47 -0700 Subject: [PATCH 87/88] errno: Add ENOTCONN constant. This is present in core MicroPython so needs to be added here. Signed-off-by: Nick Budak --- python-stdlib/errno/errno.py | 1 + python-stdlib/errno/manifest.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python-stdlib/errno/errno.py b/python-stdlib/errno/errno.py index 05441b69c..c513a7f14 100644 --- a/python-stdlib/errno/errno.py +++ b/python-stdlib/errno/errno.py @@ -34,5 +34,6 @@ ERANGE = 34 # Math result not representable EAFNOSUPPORT = 97 # Address family not supported by protocol ECONNRESET = 104 # Connection timed out +ENOTCONN = 107 # Not connected ETIMEDOUT = 110 # Connection timed out EINPROGRESS = 115 # Operation now in progress diff --git a/python-stdlib/errno/manifest.py b/python-stdlib/errno/manifest.py index a1e1e6c7c..075d3403d 100644 --- a/python-stdlib/errno/manifest.py +++ b/python-stdlib/errno/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.4") +metadata(version="0.2.0") module("errno.py") From 6e24cffe958df397c38e7b3085c16ae5d051cc92 Mon Sep 17 00:00:00 2001 From: Nick Budak Date: Wed, 4 Jun 2025 14:41:33 -0700 Subject: [PATCH 88/88] logging: Allow logging.exception helper to handle tracebacks. Although `Logger.exception` supports passing exception info with `exc_info`, when you use `logging.exception` keyword arguments are not forwarded to the root logger, which makes passing `exc_info` raise `TypeError`. Signed-off-by: Nick Budak --- python-stdlib/logging/logging.py | 4 ++-- python-stdlib/logging/manifest.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python-stdlib/logging/logging.py b/python-stdlib/logging/logging.py index f4874df7d..551bf7152 100644 --- a/python-stdlib/logging/logging.py +++ b/python-stdlib/logging/logging.py @@ -202,8 +202,8 @@ def critical(msg, *args): getLogger().critical(msg, *args) -def exception(msg, *args): - getLogger().exception(msg, *args) +def exception(msg, *args, exc_info=True): + getLogger().exception(msg, *args, exc_info=exc_info) def shutdown(): diff --git a/python-stdlib/logging/manifest.py b/python-stdlib/logging/manifest.py index d9f0ee886..c2614e9ea 100644 --- a/python-stdlib/logging/manifest.py +++ b/python-stdlib/logging/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.6.1") +metadata(version="0.6.2") module("logging.py")