diff --git a/.github/workflows/build_packages.yml b/.github/workflows/build_packages.yml index 7b79225ce..a89658e2f 100644 --- a/.github/workflows/build_packages.yml +++ b/.github/workflows/build_packages.yml @@ -2,15 +2,28 @@ name: Build all packages on: [push, pull_request] +env: + PACKAGE_INDEX_PATH: /tmp/micropython-lib-deploy + jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 - name: Setup environment run: source tools/ci.sh && ci_build_packages_setup - name: Check manifest files run: source tools/ci.sh && ci_build_packages_check_manifest - name: Compile package index run: source tools/ci.sh && ci_build_packages_compile_index + - name: Compile package examples + run: source tools/ci.sh && ci_build_packages_examples + - name: Publish packages for branch + if: vars.MICROPY_PUBLISH_MIP_INDEX && github.event_name == 'push' && ! github.event.deleted + run: source tools/ci.sh && ci_push_package_index + - name: Upload packages as artifact + uses: actions/upload-artifact@v4 + with: + name: packages-${{ github.sha }} + path: ${{ env.PACKAGE_INDEX_PATH }} diff --git a/.github/workflows/cleanup_published_packages.yml b/.github/workflows/cleanup_published_packages.yml new file mode 100644 index 000000000..040b09ff4 --- /dev/null +++ b/.github/workflows/cleanup_published_packages.yml @@ -0,0 +1,12 @@ +name: Cleanup published packages + +on: delete + +jobs: + cleanup: + runs-on: ubuntu-latest + if: vars.MICROPY_PUBLISH_MIP_INDEX + steps: + - uses: actions/checkout@v3 + - name: Clean up published files + run: source tools/ci.sh && ci_cleanup_package_index ${{ github.event.ref }} diff --git a/.github/workflows/code_formatting.yml b/.github/workflows/code_formatting.yml deleted file mode 100644 index aab347d78..000000000 --- a/.github/workflows/code_formatting.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Check code formatting - -on: [push, pull_request] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - - name: Install packages - run: source tools/ci.sh && ci_code_formatting_setup - - name: Run code formatting - run: source tools/ci.sh && ci_code_formatting_run - - name: Check code formatting - run: git diff --exit-code diff --git a/.github/workflows/commit_formatting.yml b/.github/workflows/commit_formatting.yml new file mode 100644 index 000000000..a651f8a13 --- /dev/null +++ b/.github/workflows/commit_formatting.yml @@ -0,0 +1,18 @@ +name: Check commit message formatting + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: '100' + - uses: actions/setup-python@v4 + - name: Check commit message formatting + run: source tools/ci.sh && ci_commit_formatting_run diff --git a/.github/workflows/package_tests.yml b/.github/workflows/package_tests.yml new file mode 100644 index 000000000..5e503509e --- /dev/null +++ b/.github/workflows/package_tests.yml @@ -0,0 +1,16 @@ +name: Package tests + +on: [push, pull_request] + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + - name: Setup environment + run: source tools/ci.sh && ci_package_tests_setup_micropython + - name: Setup libraries + run: source tools/ci.sh && ci_package_tests_setup_lib + - name: Run tests + run: source tools/ci.sh && ci_package_tests_run diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml new file mode 100644 index 000000000..b347e34ee --- /dev/null +++ b/.github/workflows/ruff.yml @@ -0,0 +1,12 @@ +# https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python +name: Python code lint and formatting with ruff +on: [push, pull_request] +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + # Version should be kept in sync with .pre-commit_config.yaml & also micropython + - run: pip install --user ruff==0.11.6 + - run: ruff check --output-format=github . + - run: ruff format --diff . diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..05f5d3df0 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +repos: + - repo: local + hooks: + - id: verifygitlog + name: MicroPython git commit message format checker + entry: tools/verifygitlog.py --check-file --ignore-rebase + language: python + verbose: true + stages: [commit-msg] + - repo: https://github.com/charliermarsh/ruff-pre-commit + # Version should be kept in sync with .github/workflows/ruff.yml & also micropython + rev: v0.11.6 + hooks: + - id: ruff + id: ruff-format diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 715477171..61a49101e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,11 +25,21 @@ or packages from micropython-lib, please post at the ### Pull requests The same rules for commit messages, signing-off commits, and commit structure -apply as for the main MicroPython repository. All Python code is formatted -using `black`. See [`tools/codeformat.py`](tools/codeformat.py) to apply -`black` automatically before submitting a PR. +apply [as for the main MicroPython repository](https://github.com/micropython/micropython/blob/master/CODECONVENTIONS.md). -There are some specific conventions and guidelines for micropython-lib: +All Python code is formatted using the [black](https://github.com/psf/black) +tool. You can run [`tools/codeformat.py`](tools/codeformat.py) to apply +`black` automatically before submitting a PR. The GitHub CI will also run the +[ruff](https://github.com/astral-sh/ruff) tool to apply further "linting" +checks. + +Similar to the main repository, a configuration is provided for the +[pre-commit](https://pre-commit.com/) tool to apply `black` code formatting +rules and run `ruff` automatically. See the documentation for using pre-commit +in [the code conventions document](https://github.com/micropython/micropython/blob/master/CODECONVENTIONS.md#automatic-pre-commit-hooks) + +In addition to the conventions from the main repository, there are some +specific conventions and guidelines for micropython-lib: * The first line of the commit message should start with the name of the package, followed by a short description of the commit. Package names are @@ -69,3 +79,54 @@ There are some specific conventions and guidelines for micropython-lib: * When porting an existing third-party package, please ensure that the source license is compatible. + +* To make it easier for others to install packages directly from your PR before + it is merged, consider opting-in to automatic package publishing (see + [Publishing packages from forks](#publishing-packages-from-forks)). If you do + this, consider quoting the [commands to install + packages](README.md#installing-packages-from-forks) in your Pull Request + description. + +### Publishing packages from forks + +You can easily publish the packages from your micropython-lib +[fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks) +by opting in to a system based on [GitHub +Actions](https://docs.github.com/en/actions) and [GitHub +Pages](https://docs.github.com/en/pages): + +1. Open your fork's repository in the GitHub web interface. +2. Navigate to "Settings" -> "Secrets and variables" -> "Actions" -> "Variables". +3. Click "New repository variable" +4. Create a variable named `MICROPY_PUBLISH_MIP_INDEX` with value `true` (or any + "truthy" value). +5. The settings for GitHub Actions and GitHub Pages features should not need to + be changed from the repository defaults, unless you've explicitly disabled + Actions or Pages in your fork. + +The next time you push commits to a branch in your fork, GitHub Actions will run +an additional step in the "Build All Packages" workflow named "Publish Packages +for branch". This step runs in *your fork*, but if you open a pull request then +this workflow is not shown in the Pull Request's "Checks". These run in the +upstream repository. Navigate to your fork's Actions tab in order to see +the additional "Publish Packages for branch" step. + +Anyone can then install these packages as described under [Installing packages +from forks](README.md#installing-packages-from-forks). + +The exact command is also quoted in the GitHub Actions log in your fork's +Actions for the "Publish Packages for branch" step of "Build All Packages". + +#### Opting Back Out + +To opt-out again, delete the `MICROPY_PUBLISH_MIP_INDEX` variable and +(optionally) delete the `gh-pages` branch from your fork. + +*Note*: While enabled, all micropython-lib packages will be published each time +a change is pushed to any branch in your fork. A commit is added to the +`gh-pages` branch each time. In a busy repository, the `gh-pages` branch may +become quite large. The actual `.git` directory size on disk should still be +quite small, as most of the content will be duplicated. If you're worried that +the `gh-pages` branch has become too large then you can always delete this +branch from GitHub. GitHub Actions will create a new `gh-pages` branch the next +time you push a change. diff --git a/README.md b/README.md index c47c0acf9..73417b965 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,37 @@ Note that unlike the other three approaches based on `mip` or `manifest.py`, you will need to manually resolve dependencies. You can inspect the relevant `manifest.py` file to view the list of dependencies for a given package. +## Installing packages from forks + +It is possible to use the `mpremote mip install` or `mip.install()` methods to +install packages built from a +[fork](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks) +of micropython-lib, if the fork's owner has opted in. + +This can be useful to install packages from a pending Pull Request, for example. + +First, the owner of the fork must opt-in as described under +[Publishing packages from forks](CONTRIBUTING.md#publishing-packages-from-forks). + +After this has happened, each time someone pushes to a branch in that fork then +GitHub Actions will automatically publish the packages to a GitHub Pages site. + +To install these packages, use commands such as: + +```bash +$ mpremote connect /dev/ttyUSB0 mip install --index https://USERNAME.github.io/micropython-lib/mip/BRANCH_NAME PACKAGE_NAME +``` + +Or from a networked device: + +```py +import mip +mip.install(PACKAGE_NAME, index="https://username.github.io/micropython-lib/mip/BRANCH_NAME") +``` + +(Where `USERNAME`, `BRANCH_NAME` and `PACKAGE_NAME` are replaced with the owner +of the fork, the branch the packages were built from, and the package name.) + ## Contributing We use [GitHub Discussions](https://github.com/micropython/micropython/discussions) diff --git a/micropython/aioespnow/README.md b/micropython/aioespnow/README.md new file mode 100644 index 000000000..9774d19c3 --- /dev/null +++ b/micropython/aioespnow/README.md @@ -0,0 +1,91 @@ +# `aioespnow` + +A supplementary module which extends the micropython `espnow` module to provide +`asyncio` support. + +- Asyncio support is available on all ESP32 targets as well as those ESP8266 +boards which include the `asyncio` module (ie. ESP8266 devices with at least +2MB flash storage). + +## API reference + +- class `AIOESPNow()`: inherits all the methods of the `ESPNow` class and + extends the interface with the following async methods: + + - `async AIOESPNow.arecv()` + + Asyncio support for ESPNow.recv(). Note that this method does not take a + timeout value as argument. + + - `async AIOESPNow.airecv()` + + Asyncio support for ESPNow.irecv(). Use this method to reduce memory + fragmentation, as it will reuse common storage for each new message + received, whereas the `arecv()` method will allocate new memory for every + message received. + + - `async AIOESPNow.asend(mac, msg, sync=True)` + - `async AIOESPNow.asend(msg)` + + Asyncio support for ESPNow.send(). + + - `__aiter__()/async __anext__()` + + AIOESPNow also supports reading incoming messages by asynchronous + iteration using `async for`, eg: + + ```python + e = AIOESPNow() + e.active(True) + async def recv_till_halt(e): + async for mac, msg in e: + print(mac, msg) + if msg == b'halt': + break + asyncio.run(recv_till_halt(e)) + ``` + +## Example Usage + +A small async server example:: + +```python + import network + import aioespnow + import asyncio + + # A WLAN interface must be active to send()/recv() + network.WLAN(network.WLAN.IF_STA).active(True) + + e = aioespnow.AIOESPNow() # Returns AIOESPNow enhanced with async support + e.active(True) + peer = b'\xbb\xbb\xbb\xbb\xbb\xbb' + e.add_peer(peer) + + # Send a periodic ping to a peer + async def heartbeat(e, peer, period=30): + while True: + if not await e.asend(peer, b'ping'): + print("Heartbeat: peer not responding:", peer) + else: + print("Heartbeat: ping", peer) + await asyncio.sleep(period) + + # Echo any received messages back to the sender + async def echo_server(e): + async for mac, msg in e: + print("Echo:", msg) + try: + await e.asend(mac, msg) + except OSError as err: + if len(err.args) > 1 and err.args[1] == 'ESP_ERR_ESPNOW_NOT_FOUND': + e.add_peer(mac) + await e.asend(mac, msg) + + async def main(e, peer, timeout, period): + asyncio.create_task(heartbeat(e, peer, period)) + asyncio.create_task(echo_server(e)) + await asyncio.sleep(timeout) + + asyncio.run(main(e, peer, 120, 10)) +``` diff --git a/micropython/aioespnow/aioespnow.py b/micropython/aioespnow/aioespnow.py new file mode 100644 index 000000000..dec925de2 --- /dev/null +++ b/micropython/aioespnow/aioespnow.py @@ -0,0 +1,31 @@ +# aioespnow module for MicroPython on ESP32 and ESP8266 +# MIT license; Copyright (c) 2022 Glenn Moloney @glenn20 + +import asyncio +import espnow + + +# Modelled on the asyncio.Stream class (extmod/asyncio/stream.py) +# NOTE: Relies on internal implementation of asyncio.core (_io_queue) +class AIOESPNow(espnow.ESPNow): + # Read one ESPNow message + async def arecv(self): + yield asyncio.core._io_queue.queue_read(self) + return self.recv(0) # type: ignore[misc] + + async def airecv(self): + yield asyncio.core._io_queue.queue_read(self) + return self.irecv(0) # type: ignore[misc] + + async def asend(self, mac, msg=None, sync=None): + if msg is None: + msg, mac = mac, None # If msg is None: swap mac and msg + yield asyncio.core._io_queue.queue_write(self) + return self.send(mac, msg, sync) # type: ignore[misc] + + # "async for" support + def __aiter__(self): + return self + + async def __anext__(self): + return await self.airecv() diff --git a/micropython/aioespnow/manifest.py b/micropython/aioespnow/manifest.py new file mode 100644 index 000000000..a91e48da6 --- /dev/null +++ b/micropython/aioespnow/manifest.py @@ -0,0 +1,6 @@ +metadata( + description="Extends the micropython espnow module with methods to support asyncio.", + version="0.1.0", +) + +module("aioespnow.py") diff --git a/micropython/aiorepl/README.md b/micropython/aiorepl/README.md index 2c3ed843f..c1c08b899 100644 --- a/micropython/aiorepl/README.md +++ b/micropython/aiorepl/README.md @@ -21,7 +21,7 @@ To use this library, you need to import the library and then start the REPL task For example, in main.py: ```py -import uasyncio as asyncio +import asyncio import aiorepl async def demo(): @@ -50,11 +50,17 @@ async def main(): asyncio.run(main()) ``` -The optional globals passed to `task([globals])` allows you to specify what -will be in scope for the REPL. By default it uses `__main__`, which is the -same scope as the regular REPL (and `main.py`). In the example above, the -REPL will be able to call the `demo()` function as well as get/set the -`state` variable. +An optional globals dictionary can be passed to `aiorepl.task()`, which allows +you to specify what will be in scope for the REPL. By default it uses the +globals dictionary from the `__main__` module, which is the same scope as the +regular REPL (and `main.py`). In the example above, the REPL will be able to +call the `demo()` function as well as get/set the `state` variable. + +You can also provide your own dictionary, e.g. `aiorepl.task({"obj": obj })`, +or use the globals dict from the current module, e.g. +`aiorepl.task(globals())`. Note that you cannot use a class instance's members +dictionary, e.g. `aiorepl.task(obj.__dict__)`, as this is read-only in +MicroPython. Instead of the regular `>>> ` prompt, the asyncio REPL will show `--> `. diff --git a/micropython/aiorepl/aiorepl.py b/micropython/aiorepl/aiorepl.py index 62f54c5c9..3f437459d 100644 --- a/micropython/aiorepl/aiorepl.py +++ b/micropython/aiorepl/aiorepl.py @@ -1,10 +1,11 @@ # MIT license; Copyright (c) 2022 Jim Mussared import micropython +from micropython import const import re import sys import time -import uasyncio as asyncio +import asyncio # Import statement (needs to be global, and does not return). _RE_IMPORT = re.compile("^import ([^ ]+)( as ([^ ]+))?") @@ -18,6 +19,13 @@ _HISTORY_LIMIT = const(5 + 1) +CHAR_CTRL_A = const(1) +CHAR_CTRL_B = const(2) +CHAR_CTRL_C = const(3) +CHAR_CTRL_D = const(4) +CHAR_CTRL_E = const(5) + + async def execute(code, g, s): if not code.strip(): return @@ -26,23 +34,23 @@ async def execute(code, g, s): if "await " in code: # Execute the code snippet in an async context. if m := _RE_IMPORT.match(code) or _RE_FROM_IMPORT.match(code): - code = f"global {m.group(3) or m.group(1)}\n {code}" + code = "global {}\n {}".format(m.group(3) or m.group(1), code) elif m := _RE_GLOBAL.match(code): - code = f"global {m.group(1)}\n {code}" + code = "global {}\n {}".format(m.group(1), code) elif not _RE_ASSIGN.search(code): - code = f"return {code}" + code = "return {}".format(code) - code = f""" -import uasyncio as asyncio + code = """ +import asyncio async def __code(): - {code} + {} __exec_task = asyncio.create_task(__code()) -""" +""".format(code) async def kbd_intr_task(exec_task, s): while True: - if ord(await s.read(1)) == 0x03: + if ord(await s.read(1)) == CHAR_CTRL_C: exec_task.cancel() return @@ -81,7 +89,7 @@ async def kbd_intr_task(exec_task, s): micropython.kbd_intr(-1) except Exception as err: - print(f"{type(err).__name__}: {err}") + print("{}: {}".format(type(err).__name__, err)) # REPL task. Invoke this with an optional mutable globals dict. @@ -101,7 +109,9 @@ async def task(g=None, prompt="--> "): while True: hist_b = 0 # How far back in the history are we currently. sys.stdout.write(prompt) - cmd = "" + cmd: str = "" + paste = False + curs = 0 # cursor offset from end of cmd buffer while True: b = await s.read(1) pc = c # save previous character @@ -111,11 +121,19 @@ async def task(g=None, prompt="--> "): if c < 0x20 or c > 0x7E: if c == 0x0A: # LF + if paste: + sys.stdout.write(b) + cmd += b + continue # If the previous character was also LF, and was less # than 20 ms ago, this was likely due to CRLF->LFLF # conversion, so ignore this linefeed. if pc == 0x0A and time.ticks_diff(t, pt) < 20: continue + if curs: + # move cursor to end of the line + sys.stdout.write("\x1b[{}C".format(curs)) + curs = 0 sys.stdout.write("\n") if cmd: # Push current command. @@ -132,31 +150,45 @@ async def task(g=None, prompt="--> "): elif c == 0x08 or c == 0x7F: # Backspace. if cmd: - cmd = cmd[:-1] - sys.stdout.write("\x08 \x08") - elif c == 0x02: - # Ctrl-B + if curs: + cmd = "".join((cmd[: -curs - 1], cmd[-curs:])) + sys.stdout.write( + "\x08\x1b[K" + ) # move cursor back, erase to end of line + sys.stdout.write(cmd[-curs:]) # redraw line + sys.stdout.write("\x1b[{}D".format(curs)) # reset cursor location + else: + cmd = cmd[:-1] + sys.stdout.write("\x08 \x08") + elif c == CHAR_CTRL_A: + await raw_repl(s, g) + break + elif c == CHAR_CTRL_B: continue - elif c == 0x03: - # Ctrl-C - if pc == 0x03 and time.ticks_diff(t, pt) < 20: - # Two very quick Ctrl-C (faster than a human - # typing) likely means mpremote trying to - # escape. - asyncio.new_event_loop() - return + elif c == CHAR_CTRL_C: + if paste: + break sys.stdout.write("\n") break - elif c == 0x04: - # Ctrl-D + elif c == CHAR_CTRL_D: + if paste: + result = await execute(cmd, g, s) + if result is not None: + sys.stdout.write(repr(result)) + sys.stdout.write("\n") + break + sys.stdout.write("\n") # Shutdown asyncio. asyncio.new_event_loop() return + elif c == CHAR_CTRL_E: + sys.stdout.write("paste mode; Ctrl-C to cancel, Ctrl-D to finish\n===\n") + paste = True elif c == 0x1B: # Start of escape sequence. key = await s.read(2) - if key in ("[A", "[B"): + if key in ("[A", "[B"): # up, down # Stash the current command. hist[(hist_i - hist_b) % _HISTORY_LIMIT] = cmd # Clear current command. @@ -172,12 +204,122 @@ async def task(g=None, prompt="--> "): # Update current command. cmd = hist[(hist_i - hist_b) % _HISTORY_LIMIT] sys.stdout.write(cmd) + elif key == "[D": # left + if curs < len(cmd) - 1: + curs += 1 + sys.stdout.write("\x1b") + sys.stdout.write(key) + elif key == "[C": # right + if curs: + curs -= 1 + sys.stdout.write("\x1b") + sys.stdout.write(key) + elif key == "[H": # home + pcurs = curs + curs = len(cmd) + sys.stdout.write("\x1b[{}D".format(curs - pcurs)) # move cursor left + elif key == "[F": # end + pcurs = curs + curs = 0 + sys.stdout.write("\x1b[{}C".format(pcurs)) # move cursor right else: # sys.stdout.write("\\x") # sys.stdout.write(hex(c)) pass else: - sys.stdout.write(b) - cmd += b + if curs: + # inserting into middle of line + cmd = "".join((cmd[:-curs], b, cmd[-curs:])) + sys.stdout.write(cmd[-curs - 1 :]) # redraw line to end + sys.stdout.write("\x1b[{}D".format(curs)) # reset cursor location + else: + sys.stdout.write(b) + cmd += b finally: micropython.kbd_intr(3) + + +async def raw_paste(s, g, window=512): + sys.stdout.write("R\x01") # supported + sys.stdout.write(bytearray([window & 0xFF, window >> 8, 0x01]).decode()) + eof = False + idx = 0 + buff = bytearray(window) + file = b"" + while not eof: + for idx in range(window): + b = await s.read(1) + c = ord(b) + if c == CHAR_CTRL_C or c == CHAR_CTRL_D: + # end of file + sys.stdout.write(chr(CHAR_CTRL_D)) + if c == CHAR_CTRL_C: + raise KeyboardInterrupt + file += buff[:idx] + eof = True + break + buff[idx] = c + + if not eof: + file += buff + sys.stdout.write("\x01") # indicate window available to host + + return file + + +async def raw_repl(s: asyncio.StreamReader, g: dict): + heading = "raw REPL; CTRL-B to exit\n" + line = "" + sys.stdout.write(heading) + + while True: + line = "" + sys.stdout.write(">") + while True: + b = await s.read(1) + c = ord(b) + if c == CHAR_CTRL_A: + rline = line + line = "" + + if len(rline) == 2 and ord(rline[0]) == CHAR_CTRL_E: + if rline[1] == "A": + line = await raw_paste(s, g) + break + else: + # reset raw REPL + sys.stdout.write(heading) + sys.stdout.write(">") + continue + elif c == CHAR_CTRL_B: + # exit raw REPL + sys.stdout.write("\n") + return 0 + elif c == CHAR_CTRL_C: + # clear line + line = "" + elif c == CHAR_CTRL_D: + # entry finished + # indicate reception of command + sys.stdout.write("OK") + break + else: + # let through any other raw 8-bit value + line += b + + if len(line) == 0: + # Normally used to trigger soft-reset but stay in raw mode. + # Fake it for aiorepl / mpremote. + sys.stdout.write("Ignored: soft reboot\n") + sys.stdout.write(heading) + + try: + result = exec(line, g) + if result is not None: + sys.stdout.write(repr(result)) + sys.stdout.write(chr(CHAR_CTRL_D)) + except Exception as ex: + print(line) + sys.stdout.write(chr(CHAR_CTRL_D)) + sys.print_exception(ex, sys.stdout) + sys.stdout.write(chr(CHAR_CTRL_D)) diff --git a/micropython/aiorepl/manifest.py b/micropython/aiorepl/manifest.py index 5cce6c796..0fcc21849 100644 --- a/micropython/aiorepl/manifest.py +++ b/micropython/aiorepl/manifest.py @@ -1,5 +1,5 @@ metadata( - version="0.1", + version="0.2.0", description="Provides an asynchronous REPL that can run concurrently with an asyncio, also allowing await expressions.", ) diff --git a/micropython/bluetooth/aioble-central/manifest.py b/micropython/bluetooth/aioble-central/manifest.py index beec50460..ed61ec9d7 100644 --- a/micropython/bluetooth/aioble-central/manifest.py +++ b/micropython/bluetooth/aioble-central/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0") +metadata(version="0.3.0") require("aioble-core") diff --git a/micropython/bluetooth/aioble-client/manifest.py b/micropython/bluetooth/aioble-client/manifest.py index eb79c6d33..163cbe23d 100644 --- a/micropython/bluetooth/aioble-client/manifest.py +++ b/micropython/bluetooth/aioble-client/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0") +metadata(version="0.3.0") require("aioble-core") diff --git a/micropython/bluetooth/aioble-core/manifest.py b/micropython/bluetooth/aioble-core/manifest.py index 2448769e6..e040f1076 100644 --- a/micropython/bluetooth/aioble-core/manifest.py +++ b/micropython/bluetooth/aioble-core/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0") +metadata(version="0.4.0") package( "aioble", diff --git a/micropython/bluetooth/aioble-peripheral/manifest.py b/micropython/bluetooth/aioble-peripheral/manifest.py index dd4dd122d..0aec4d21e 100644 --- a/micropython/bluetooth/aioble-peripheral/manifest.py +++ b/micropython/bluetooth/aioble-peripheral/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0") +metadata(version="0.2.1") require("aioble-core") diff --git a/micropython/bluetooth/aioble-server/manifest.py b/micropython/bluetooth/aioble-server/manifest.py index a9676204d..c5b12ffbd 100644 --- a/micropython/bluetooth/aioble-server/manifest.py +++ b/micropython/bluetooth/aioble-server/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.2.0") +metadata(version="0.4.1") require("aioble-core") diff --git a/micropython/bluetooth/aioble/README.md b/micropython/bluetooth/aioble/README.md index 6b6b204f6..83ae00209 100644 --- a/micropython/bluetooth/aioble/README.md +++ b/micropython/bluetooth/aioble/README.md @@ -70,7 +70,7 @@ Alternatively, install the `aioble` package, which will install everything. Usage ----- -Passive scan for nearby devices for 5 seconds: (Observer) +#### Passive scan for nearby devices for 5 seconds: (Observer) ```py async with aioble.scan(duration_ms=5000) as scanner: @@ -87,7 +87,7 @@ async with aioble.scan(duration_ms=5000, interval_us=30000, window_us=30000, act print(result, result.name(), result.rssi, result.services()) ``` -Connect to a peripheral device: (Central) +#### Connect to a peripheral device: (Central) ```py # Either from scan result @@ -101,14 +101,14 @@ except asyncio.TimeoutError: print('Timeout') ``` -Register services and wait for connection: (Peripheral, Server) +#### Register services and wait for connection: (Peripheral, Server) ```py _ENV_SENSE_UUID = bluetooth.UUID(0x181A) _ENV_SENSE_TEMP_UUID = bluetooth.UUID(0x2A6E) _GENERIC_THERMOMETER = const(768) -_ADV_INTERVAL_MS = const(250000) +_ADV_INTERVAL_US = const(250000) temp_service = aioble.Service(_ENV_SENSE_UUID) temp_char = aioble.Characteristic(temp_service, _ENV_SENSE_TEMP_UUID, read=True, notify=True) @@ -117,7 +117,7 @@ aioble.register_services(temp_service) while True: connection = await aioble.advertise( - _ADV_INTERVAL_MS, + _ADV_INTERVAL_US, name="temp-sense", services=[_ENV_SENSE_UUID], appearance=_GENERIC_THERMOMETER, @@ -126,30 +126,95 @@ while True: print("Connection from", device) ``` -Update characteristic value: (Server) +#### Update characteristic value: (Server) ```py +# Write the local value. temp_char.write(b'data') +``` + +```py +# Write the local value and notify/indicate subscribers. +temp_char.write(b'data', send_update=True) +``` + +#### Send notifications: (Server) -temp_char.notify(b'optional data') +```py +# Notify with the current value. +temp_char.notify(connection) +``` -await temp_char.indicate(timeout_ms=2000) +```py +# Notify with a custom value. +temp_char.notify(connection, b'optional data') ``` -Query the value of a characteristic: (Client) +#### Send indications: (Server) + +```py +# Indicate with current value. +await temp_char.indicate(connection, timeout_ms=2000) +``` + +```py +# Indicate with custom value. +await temp_char.indicate(connection, b'optional data', timeout_ms=2000) +``` + +This will raise `GattError` if the indication is not acknowledged. + +#### Wait for a write from the client: (Server) + +```py +# Normal characteristic, returns the connection that did the write. +connection = await char.written(timeout_ms=2000) +``` + +```py +# Characteristic with capture enabled, also returns the value. +char = Characteristic(..., capture=True) +connection, data = await char.written(timeout_ms=2000) +``` + +#### Query the value of a characteristic: (Client) ```py temp_service = await connection.service(_ENV_SENSE_UUID) temp_char = await temp_service.characteristic(_ENV_SENSE_TEMP_UUID) data = await temp_char.read(timeout_ms=1000) +``` + +#### Wait for a notification/indication: (Client) + +```py +# Notification +data = await temp_char.notified(timeout_ms=1000) +``` +```py +# Indication +data = await temp_char.indicated(timeout_ms=1000) +``` + +#### Subscribe to a characteristic: (Client) + +```py +# Subscribe for notification. await temp_char.subscribe(notify=True) while True: data = await temp_char.notified() ``` -Open L2CAP channels: (Listener) +```py +# Subscribe for indication. +await temp_char.subscribe(indicate=True) +while True: + data = await temp_char.indicated() +``` + +#### Open L2CAP channels: (Listener) ```py channel = await connection.l2cap_accept(_L2CAP_PSN, _L2CAP_MTU) @@ -158,7 +223,7 @@ n = channel.recvinto(buf) channel.send(b'response') ``` -Open L2CAP channels: (Initiator) +#### Open L2CAP channels: (Initiator) ```py channel = await connection.l2cap_connect(_L2CAP_PSN, _L2CAP_MTU) diff --git a/micropython/bluetooth/aioble/aioble/central.py b/micropython/bluetooth/aioble/aioble/central.py index 46da907a7..131b1e0db 100644 --- a/micropython/bluetooth/aioble/aioble/central.py +++ b/micropython/bluetooth/aioble/aioble/central.py @@ -6,7 +6,7 @@ import bluetooth import struct -import uasyncio as asyncio +import asyncio from .core import ( ensure_active, @@ -33,6 +33,7 @@ _ADV_TYPE_FLAGS = const(0x01) _ADV_TYPE_NAME = const(0x09) +_ADV_TYPE_SHORT_NAME = const(0x08) _ADV_TYPE_UUID16_INCOMPLETE = const(0x2) _ADV_TYPE_UUID16_COMPLETE = const(0x3) _ADV_TYPE_UUID32_INCOMPLETE = const(0x4) @@ -103,7 +104,9 @@ async def _cancel_pending(): # Start connecting to a peripheral. # Call device.connect() rather than using method directly. -async def _connect(connection, timeout_ms): +async def _connect( + connection, timeout_ms, scan_duration_ms, min_conn_interval_us, max_conn_interval_us +): device = connection.device if device in _connecting: return @@ -121,7 +124,13 @@ async def _connect(connection, timeout_ms): try: with DeviceTimeout(None, timeout_ms): - ble.gap_connect(device.addr_type, device.addr) + ble.gap_connect( + device.addr_type, + device.addr, + scan_duration_ms, + min_conn_interval_us, + max_conn_interval_us, + ) # Wait for the connected IRQ. await connection._event.wait() @@ -187,19 +196,21 @@ def _decode_field(self, *adv_type): yield payload[i + 2 : i + payload[i] + 1] i += 1 + payload[i] - # Returns the value of the advertised name, otherwise empty string. + # Returns the value of the complete (or shortened) advertised name, if available. def name(self): - for n in self._decode_field(_ADV_TYPE_NAME): + for n in self._decode_field(_ADV_TYPE_NAME, _ADV_TYPE_SHORT_NAME): return str(n, "utf-8") if n else "" # Generator that enumerates the service UUIDs that are advertised. def services(self): - for u in self._decode_field(_ADV_TYPE_UUID16_INCOMPLETE, _ADV_TYPE_UUID16_COMPLETE): - yield bluetooth.UUID(struct.unpack(" maxval: - value = maxval + value = max(value, minval) + value = min(value, maxval) return value target = limit((16 + (target * 2) // 3), 0, 15) @@ -718,8 +715,7 @@ def logb(value, limit): while value > 1: value >>= 1 lb += 1 - if lb > limit: - lb = limit + lb = min(lb, limit) return lb attack = logb(attack / 6, 7) diff --git a/micropython/drivers/display/lcd160cr/lcd160cr.py b/micropython/drivers/display/lcd160cr/lcd160cr.py index f792418aa..177c6fea3 100644 --- a/micropython/drivers/display/lcd160cr/lcd160cr.py +++ b/micropython/drivers/display/lcd160cr/lcd160cr.py @@ -2,9 +2,10 @@ # MIT license; Copyright (c) 2017 Damien P. George from micropython import const +import machine from utime import sleep_ms from ustruct import calcsize, pack_into -import uerrno, machine +import errno # for set_orient PORTRAIT = const(0) @@ -109,7 +110,7 @@ def _waitfor(self, n, buf): return t -= 1 sleep_ms(1) - raise OSError(uerrno.ETIMEDOUT) + raise OSError(errno.ETIMEDOUT) def oflush(self, n=255): t = 5000 @@ -120,7 +121,7 @@ def oflush(self, n=255): return t -= 1 machine.idle() - raise OSError(uerrno.ETIMEDOUT) + raise OSError(errno.ETIMEDOUT) def iflush(self): t = 5000 @@ -130,7 +131,7 @@ def iflush(self): return t -= 1 sleep_ms(1) - raise OSError(uerrno.ETIMEDOUT) + raise OSError(errno.ETIMEDOUT) #### MISC METHODS #### @@ -188,10 +189,8 @@ def clip_line(c, w, h): c[3] = h - 1 else: if c[0] == c[2]: - if c[1] < 0: - c[1] = 0 - if c[3] < 0: - c[3] = 0 + c[1] = max(c[1], 0) + c[3] = max(c[3], 0) else: if c[3] < c[1]: c[0], c[2] = c[2], c[0] @@ -253,7 +252,7 @@ def get_pixel(self, x, y): return self.buf[3][1] | self.buf[3][2] << 8 t -= 1 sleep_ms(1) - raise OSError(uerrno.ETIMEDOUT) + raise OSError(errno.ETIMEDOUT) def get_line(self, x, y, buf): l = len(buf) // 2 @@ -267,7 +266,7 @@ def get_line(self, x, y, buf): return t -= 1 sleep_ms(1) - raise OSError(uerrno.ETIMEDOUT) + raise OSError(errno.ETIMEDOUT) def screen_dump(self, buf, x=0, y=0, w=None, h=None): if w is None: diff --git a/micropython/drivers/display/lcd160cr/lcd160cr_test.py b/micropython/drivers/display/lcd160cr/lcd160cr_test.py index 883c7d3b7..b2a9c0c6a 100644 --- a/micropython/drivers/display/lcd160cr/lcd160cr_test.py +++ b/micropython/drivers/display/lcd160cr/lcd160cr_test.py @@ -1,11 +1,14 @@ # Driver test for official MicroPython LCD160CR display # MIT license; Copyright (c) 2017 Damien P. George -import time, math, framebuf, lcd160cr +import framebuf +import lcd160cr +import math +import time def get_lcd(lcd): - if type(lcd) is str: + if isinstance(lcd, str): lcd = lcd160cr.LCD160CR(lcd) return lcd diff --git a/micropython/drivers/display/lcd160cr/manifest.py b/micropython/drivers/display/lcd160cr/manifest.py index 62a19eb2b..9e18a02a7 100644 --- a/micropython/drivers/display/lcd160cr/manifest.py +++ b/micropython/drivers/display/lcd160cr/manifest.py @@ -1,6 +1,3 @@ -options.defaults(test=False) +metadata(description="LCD160CR driver.", version="0.1.0") module("lcd160cr.py", opt=3) - -if options.test: - module("lcd160cr_test.py", opt=3) diff --git a/micropython/drivers/display/ssd1306/manifest.py b/micropython/drivers/display/ssd1306/manifest.py index 51580080e..80253be44 100644 --- a/micropython/drivers/display/ssd1306/manifest.py +++ b/micropython/drivers/display/ssd1306/manifest.py @@ -1 +1,3 @@ +metadata(description="SSD1306 OLED driver.", version="0.1.0") + module("ssd1306.py", opt=3) diff --git a/micropython/drivers/imu/bmi270/bmi270.py b/micropython/drivers/imu/bmi270/bmi270.py index 9e21b7395..64f819ec2 100644 --- a/micropython/drivers/imu/bmi270/bmi270.py +++ b/micropython/drivers/imu/bmi270/bmi270.py @@ -524,13 +524,13 @@ def __init__( # Sanity checks if not self._use_i2c: raise ValueError("SPI mode is not supported") - if not gyro_odr in ODR: + if gyro_odr not in ODR: raise ValueError("Invalid gyro sampling rate: %d" % gyro_odr) - if not gyro_scale in GYRO_SCALE: + if gyro_scale not in GYRO_SCALE: raise ValueError("Invalid gyro scaling: %d" % gyro_scale) - if not accel_odr in ODR: + if accel_odr not in ODR: raise ValueError("Invalid accelerometer sampling rate: %d" % accel_odr) - if not accel_scale in ACCEL_SCALE: + if accel_scale not in ACCEL_SCALE: raise ValueError("Invalid accelerometer scaling: %d" % accel_scale) if self._read_reg(_CHIP_ID) != 0x24: raise OSError("No BMI270 device was found at address 0x%x" % (self.address)) @@ -598,7 +598,7 @@ def _write_reg(self, reg, val): def _write_burst(self, reg, data, chunk=16): self._write_reg(_INIT_ADDR_0, 0) self._write_reg(_INIT_ADDR_1, 0) - for i in range(0, len(data) // chunk): + for i in range(len(data) // chunk): offs = i * chunk self._write_reg(reg, data[offs : offs + chunk]) init_addr = ((i + 1) * chunk) // 2 @@ -606,7 +606,7 @@ def _write_burst(self, reg, data, chunk=16): self._write_reg(_INIT_ADDR_1, (init_addr >> 4) & 0xFF) def _poll_reg(self, reg, mask, retry=10, delay=100): - for i in range(0, retry): + for i in range(retry): if self._read_reg(reg) & mask: return True time.sleep_ms(delay) diff --git a/micropython/drivers/imu/bmm150/bmm150.py b/micropython/drivers/imu/bmm150/bmm150.py index 12220f643..a4845c961 100644 --- a/micropython/drivers/imu/bmm150/bmm150.py +++ b/micropython/drivers/imu/bmm150/bmm150.py @@ -80,7 +80,7 @@ def __init__( # Sanity checks if not self._use_i2c: raise ValueError("SPI mode is not supported") - if not magnet_odr in _ODR: + if magnet_odr not in _ODR: raise ValueError("Invalid sampling rate: %d" % magnet_odr) # Perform soft reset, and power on. @@ -165,11 +165,8 @@ def _compensate_z(self, raw, hall): z = (z5 / (z4 * 4)) / 16 return z - def reset(self): - self._write_reg(_CMD, 0xB6) - def magnet_raw(self): - for i in range(0, 10): + for i in range(10): self._read_reg_into(_DATA, self.scratch) if self.scratch[3] & 0x1: return ( diff --git a/micropython/drivers/imu/lsm6dsox/lsm6dsox.py b/micropython/drivers/imu/lsm6dsox/lsm6dsox.py index 2e043f24c..ca1397c66 100644 --- a/micropython/drivers/imu/lsm6dsox/lsm6dsox.py +++ b/micropython/drivers/imu/lsm6dsox/lsm6dsox.py @@ -46,6 +46,7 @@ import array from micropython import const +import time _CTRL3_C = const(0x12) _CTRL1_XL = const(0x10) @@ -130,20 +131,20 @@ def __init__( accel_odr = round(accel_odr, 2) # Sanity checks - if not gyro_odr in ODR: + if gyro_odr not in ODR: raise ValueError("Invalid sampling rate: %d" % gyro_odr) - if not gyro_scale in SCALE_GYRO: + if gyro_scale not in SCALE_GYRO: raise ValueError("invalid gyro scaling: %d" % gyro_scale) - if not accel_odr in ODR: + if accel_odr not in ODR: raise ValueError("Invalid sampling rate: %d" % accel_odr) - if not accel_scale in SCALE_ACCEL: + if accel_scale not in SCALE_ACCEL: raise ValueError("invalid accelerometer scaling: %d" % accel_scale) # Soft-reset the device. self.reset() # Load and configure MLC if UCF file is provided - if ucf != None: + if ucf is not None: self.load_mlc(ucf) # Set Gyroscope datarate and scale. @@ -196,7 +197,7 @@ def _read_reg_into(self, reg, buf): def reset(self): self._write_reg(_CTRL3_C, self._read_reg(_CTRL3_C) | 0x1) - for i in range(0, 10): + for i in range(10): if (self._read_reg(_CTRL3_C) & 0x01) == 0: return time.sleep_ms(10) diff --git a/micropython/drivers/imu/lsm6dsox/lsm6dsox_mlc.py b/micropython/drivers/imu/lsm6dsox/lsm6dsox_mlc.py index ce3ff8e92..2a53b9402 100644 --- a/micropython/drivers/imu/lsm6dsox/lsm6dsox_mlc.py +++ b/micropython/drivers/imu/lsm6dsox/lsm6dsox_mlc.py @@ -17,7 +17,7 @@ def imu_int_handler(pin): INT_FLAG = True -if INT_MODE == True: +if INT_MODE is True: int_pin = Pin(24) int_pin.irq(handler=imu_int_handler, trigger=Pin.IRQ_RISING) @@ -44,5 +44,5 @@ def imu_int_handler(pin): print(UCF_LABELS[lsm.mlc_output()[0]]) else: buf = lsm.mlc_output() - if buf != None: + if buf is not None: print(UCF_LABELS[buf[0]]) diff --git a/micropython/drivers/imu/lsm6dsox/manifest.py b/micropython/drivers/imu/lsm6dsox/manifest.py index 3bf037679..346255fe7 100644 --- a/micropython/drivers/imu/lsm6dsox/manifest.py +++ b/micropython/drivers/imu/lsm6dsox/manifest.py @@ -1,2 +1,2 @@ -metadata(description="ST LSM6DSOX imu driver.", version="1.0.0") +metadata(description="ST LSM6DSOX imu driver.", version="1.0.1") module("lsm6dsox.py", opt=3) diff --git a/micropython/drivers/imu/lsm9ds1/lsm9ds1.py b/micropython/drivers/imu/lsm9ds1/lsm9ds1.py index 8042ecc6f..e5a96ad5c 100644 --- a/micropython/drivers/imu/lsm9ds1/lsm9ds1.py +++ b/micropython/drivers/imu/lsm9ds1/lsm9ds1.py @@ -43,7 +43,9 @@ print("") time.sleep_ms(100) """ + import array +from micropython import const _WHO_AM_I = const(0xF) @@ -95,19 +97,19 @@ def __init__( self.address_magnet = address_magnet # Sanity checks - if not gyro_odr in _ODR_IMU: + if gyro_odr not in _ODR_IMU: raise ValueError("Invalid gyro sampling rate: %d" % gyro_odr) - if not gyro_scale in _GYRO_SCALE: + if gyro_scale not in _GYRO_SCALE: raise ValueError("Invalid gyro scaling: %d" % gyro_scale) - if not accel_odr in _ODR_IMU: + if accel_odr not in _ODR_IMU: raise ValueError("Invalid accelerometer sampling rate: %d" % accel_odr) - if not accel_scale in _ACCEL_SCALE: + if accel_scale not in _ACCEL_SCALE: raise ValueError("Invalid accelerometer scaling: %d" % accel_scale) - if not magnet_odr in _ODR_MAGNET: + if magnet_odr not in _ODR_MAGNET: raise ValueError("Invalid magnet sampling rate: %d" % magnet_odr) - if not magnet_scale in _MAGNET_SCALE: + if magnet_scale not in _MAGNET_SCALE: raise ValueError("Invalid magnet scaling: %d" % magnet_scale) if (self.magent_id() != b"=") or (self.gyro_id() != b"h"): diff --git a/micropython/drivers/led/neopixel/manifest.py b/micropython/drivers/led/neopixel/manifest.py index 561d19574..02a319002 100644 --- a/micropython/drivers/led/neopixel/manifest.py +++ b/micropython/drivers/led/neopixel/manifest.py @@ -1 +1,3 @@ +metadata(description="WS2812/NeoPixel driver.", version="0.1.0") + module("neopixel.py", opt=3) diff --git a/micropython/drivers/radio/nrf24l01/manifest.py b/micropython/drivers/radio/nrf24l01/manifest.py index babdb7a52..24276ee4b 100644 --- a/micropython/drivers/radio/nrf24l01/manifest.py +++ b/micropython/drivers/radio/nrf24l01/manifest.py @@ -1 +1,3 @@ +metadata(description="nrf24l01 2.4GHz radio driver.", version="0.2.0") + module("nrf24l01.py", opt=3) diff --git a/micropython/drivers/radio/nrf24l01/nrf24l01.py b/micropython/drivers/radio/nrf24l01/nrf24l01.py index 76d55312f..9fbadcd60 100644 --- a/micropython/drivers/radio/nrf24l01/nrf24l01.py +++ b/micropython/drivers/radio/nrf24l01/nrf24l01.py @@ -1,5 +1,4 @@ -"""NRF24L01 driver for MicroPython -""" +"""NRF24L01 driver for MicroPython""" from micropython import const import utime @@ -130,6 +129,13 @@ def reg_write(self, reg, value): self.cs(1) return ret + def read_status(self): + self.cs(0) + # STATUS register is always shifted during command transmit + self.spi.readinto(self.buf, NOP) + self.cs(1) + return self.buf[0] + def flush_rx(self): self.cs(0) self.spi.readinto(self.buf, FLUSH_RX) @@ -220,6 +226,13 @@ def send(self, buf, timeout=500): result = None while result is None and utime.ticks_diff(utime.ticks_ms(), start) < timeout: result = self.send_done() # 1 == success, 2 == fail + + if result is None: + # timed out, cancel sending and power down the module + self.flush_tx() + self.reg_write(CONFIG, self.reg_read(CONFIG) & ~PWR_UP) + raise OSError("timed out") + if result == 2: raise OSError("send failed") @@ -227,7 +240,7 @@ def send(self, buf, timeout=500): def send_start(self, buf): # power up self.reg_write(CONFIG, (self.reg_read(CONFIG) | PWR_UP) & ~PRIM_RX) - utime.sleep_us(150) + utime.sleep_us(1500) # needs to be 1.5ms # send the data self.cs(0) self.spi.readinto(self.buf, W_TX_PAYLOAD) @@ -243,7 +256,8 @@ def send_start(self, buf): # returns None if send still in progress, 1 for success, 2 for fail def send_done(self): - if not (self.reg_read(STATUS) & (TX_DS | MAX_RT)): + status = self.read_status() + if not (status & (TX_DS | MAX_RT)): return None # tx not finished # either finished or failed: get and clear status flags, power down diff --git a/micropython/drivers/radio/nrf24l01/nrf24l01test.py b/micropython/drivers/radio/nrf24l01/nrf24l01test.py index ad3e1f67a..a0c4b76f4 100644 --- a/micropython/drivers/radio/nrf24l01/nrf24l01test.py +++ b/micropython/drivers/radio/nrf24l01/nrf24l01test.py @@ -1,7 +1,7 @@ """Test for nrf24l01 module. Portable between MicroPython targets.""" -import usys -import ustruct as struct +import sys +import struct import utime from machine import Pin, SPI, SoftSPI from nrf24l01 import NRF24L01 @@ -14,20 +14,20 @@ # initiator may be a slow device. Value tested with Pyboard, ESP32 and ESP8266. _RESPONDER_SEND_DELAY = const(10) -if usys.platform == "pyboard": +if sys.platform == "pyboard": spi = SPI(2) # miso : Y7, mosi : Y8, sck : Y6 cfg = {"spi": spi, "csn": "Y5", "ce": "Y4"} -elif usys.platform == "esp8266": # Hardware SPI +elif sys.platform == "esp8266": # Hardware SPI spi = SPI(1) # miso : 12, mosi : 13, sck : 14 cfg = {"spi": spi, "csn": 4, "ce": 5} -elif usys.platform == "esp32": # Software SPI +elif sys.platform == "esp32": # Software SPI spi = SoftSPI(sck=Pin(25), mosi=Pin(33), miso=Pin(32)) cfg = {"spi": spi, "csn": 26, "ce": 27} -elif usys.platform == "rp2": # Hardware SPI with explicit pin definitions +elif sys.platform == "rp2": # Hardware SPI with explicit pin definitions spi = SPI(0, sck=Pin(2), mosi=Pin(3), miso=Pin(4)) cfg = {"spi": spi, "csn": 5, "ce": 6} else: - raise ValueError("Unsupported platform {}".format(usys.platform)) + raise ValueError("Unsupported platform {}".format(sys.platform)) # Addresses are in little-endian format. They correspond to big-endian # 0xf0f0f0f0e1, 0xf0f0f0f0d2 diff --git a/micropython/drivers/sensor/dht/manifest.py b/micropython/drivers/sensor/dht/manifest.py index 72a4e0d24..964e8e252 100644 --- a/micropython/drivers/sensor/dht/manifest.py +++ b/micropython/drivers/sensor/dht/manifest.py @@ -1 +1,3 @@ +metadata(description="DHT11 & DHT22 temperature/humidity sensor driver.", version="0.1.0") + module("dht.py", opt=3) diff --git a/micropython/drivers/sensor/ds18x20/manifest.py b/micropython/drivers/sensor/ds18x20/manifest.py index 01f7ae035..6ced882f7 100644 --- a/micropython/drivers/sensor/ds18x20/manifest.py +++ b/micropython/drivers/sensor/ds18x20/manifest.py @@ -1,2 +1,4 @@ +metadata(description="DS18x20 temperature sensor driver.", version="0.1.0") + require("onewire") module("ds18x20.py", opt=3) diff --git a/micropython/drivers/sensor/hts221/hts221.py b/micropython/drivers/sensor/hts221/hts221.py index fec52a738..c6cd51f48 100644 --- a/micropython/drivers/sensor/hts221/hts221.py +++ b/micropython/drivers/sensor/hts221/hts221.py @@ -52,7 +52,7 @@ def __init__(self, i2c, data_rate=1, address=0x5F): # Set configuration register # Humidity and temperature average configuration - self.bus.writeto_mem(self.slv_addr, 0x10, b"\x1B") + self.bus.writeto_mem(self.slv_addr, 0x10, b"\x1b") # Set control register # PD | BDU | ODR diff --git a/micropython/drivers/sensor/hts221/manifest.py b/micropython/drivers/sensor/hts221/manifest.py index 5f1792665..d85edaac8 100644 --- a/micropython/drivers/sensor/hts221/manifest.py +++ b/micropython/drivers/sensor/hts221/manifest.py @@ -1 +1,3 @@ +metadata(description="HTS221 temperature/humidity sensor driver.", version="0.1.0") + module("hts221.py", opt=3) diff --git a/micropython/drivers/sensor/lps22h/lps22h.py b/micropython/drivers/sensor/lps22h/lps22h.py index ca29efce2..7dec72528 100644 --- a/micropython/drivers/sensor/lps22h/lps22h.py +++ b/micropython/drivers/sensor/lps22h/lps22h.py @@ -37,7 +37,9 @@ print("Pressure: %.2f hPa Temperature: %.2f C"%(lps.pressure(), lps.temperature())) time.sleep_ms(10) """ + import machine +from micropython import const _LPS22_CTRL_REG1 = const(0x10) _LPS22_CTRL_REG2 = const(0x11) diff --git a/micropython/drivers/sensor/lps22h/manifest.py b/micropython/drivers/sensor/lps22h/manifest.py index d30108d93..971cbfdcb 100644 --- a/micropython/drivers/sensor/lps22h/manifest.py +++ b/micropython/drivers/sensor/lps22h/manifest.py @@ -1 +1,3 @@ +metadata(description="LPS22H temperature/pressure sensor driver.", version="0.1.0") + module("lps22h.py", opt=3) diff --git a/micropython/drivers/storage/sdcard/manifest.py b/micropython/drivers/storage/sdcard/manifest.py index e584b97d9..cb4647eeb 100644 --- a/micropython/drivers/storage/sdcard/manifest.py +++ b/micropython/drivers/storage/sdcard/manifest.py @@ -1 +1,3 @@ +metadata(description="SDCard block device driver.", version="0.1.0") + module("sdcard.py", opt=3) diff --git a/micropython/drivers/storage/sdcard/sdtest.py b/micropython/drivers/storage/sdcard/sdtest.py index 018ef7c64..ce700e2a8 100644 --- a/micropython/drivers/storage/sdcard/sdtest.py +++ b/micropython/drivers/storage/sdcard/sdtest.py @@ -1,6 +1,8 @@ # Test for sdcard block protocol # Peter hinch 30th Jan 2016 -import os, sdcard, machine +import machine +import os +import sdcard def sdtest(): diff --git a/micropython/espflash/espflash.py b/micropython/espflash/espflash.py index 700309bd9..fbf4e1f7e 100644 --- a/micropython/espflash/espflash.py +++ b/micropython/espflash/espflash.py @@ -104,7 +104,7 @@ def _write_reg(self, addr, data, mask=0xFFFFFFFF, delay=0): raise Exception("Command ESP_WRITE_REG failed.") def _poll_reg(self, addr, flag, retry=10, delay=0.050): - for i in range(0, retry): + for i in range(retry): reg = self._read_reg(addr) if (reg & flag) == 0: break @@ -113,22 +113,22 @@ def _poll_reg(self, addr, flag, retry=10, delay=0.050): raise Exception(f"Register poll timeout. Addr: 0x{addr:02X} Flag: 0x{flag:02X}.") def _write_slip(self, pkt): - pkt = pkt.replace(b"\xDB", b"\xdb\xdd").replace(b"\xc0", b"\xdb\xdc") - self.uart.write(b"\xC0" + pkt + b"\xC0") + pkt = pkt.replace(b"\xdb", b"\xdb\xdd").replace(b"\xc0", b"\xdb\xdc") + self.uart.write(b"\xc0" + pkt + b"\xc0") self._log(pkt) def _read_slip(self): pkt = None # Find the packet start. - if self.uart.read(1) == b"\xC0": + if self.uart.read(1) == b"\xc0": pkt = bytearray() while True: b = self.uart.read(1) - if b is None or b == b"\xC0": + if b is None or b == b"\xc0": break pkt += b - pkt = pkt.replace(b"\xDB\xDD", b"\xDB").replace(b"\xDB\xDC", b"\xC0") - self._log(b"\xC0" + pkt + b"\xC0", False) + pkt = pkt.replace(b"\xdb\xdd", b"\xdb").replace(b"\xdb\xdc", b"\xc0") + self._log(b"\xc0" + pkt + b"\xc0", False) return pkt def _strerror(self, err): @@ -230,12 +230,12 @@ def flash_read_size(self): raise Exception(f"Unexpected flash size bits: 0x{flash_bits:02X}.") flash_size = 2**flash_bits - print(f"Flash size {flash_size/1024/1024} MBytes") + print(f"Flash size {flash_size / 1024 / 1024} MBytes") return flash_size def flash_attach(self): self._command(_CMD_SPI_ATTACH, struct.pack(" {seq+erase_blocks}...") @@ -289,7 +288,7 @@ def flash_write_file(self, path, blksize=0x1000): def flash_verify_file(self, path, digest=None, offset=0): if digest is None: if self.md5sum is None: - raise Exception(f"MD5 checksum missing.") + raise Exception("MD5 checksum missing.") digest = binascii.hexlify(self.md5sum.digest()) size = os.stat(path)[6] @@ -301,7 +300,7 @@ def flash_verify_file(self, path, digest=None, offset=0): if digest == data[0:32]: print("Firmware verified.") else: - raise Exception(f"Firmware verification failed.") + raise Exception("Firmware verification failed.") def reboot(self): payload = struct.pack(" +Additional Australia-specific regulatory explanation + +The LoRaWAN AU915 specifications suggest 125kHz bandwidth. To tell that it's OK +to set `bw` lower, consult the Australian [Low Interference Potential Devices +class license](https://www.legislation.gov.au/Series/F2015L01438). This class +license allows Digital Modulation Transmitters in the 915-928MHz band to +transmit up to 1W Maximum EIRP provided "*The radiated peak power spectral +density in any 3 kHz must not exceed 25 mW per 3 kHz*". + +`output_power` set to 20dBm is 100mW, over 62.5kHz bandwidth gives +1.6mW/kHz. This leaves significant headroom for antenna gain that might increase +radiated power in some directions.) + + +### Configuration Keys + +These keys can be set in the `lora_cfg` dict argument to `configure()`, +and correspond to the parameters documented in this section. + +Consult the datasheet for the LoRa modem you are using for an in-depth +description of each of these parameters. + +Values which are unset when `configure()` is called will keep their existing +values. + +#### `freq_khz` - RF Frequency +Type: `int` (recommended) or `float` (if supported by port) + +LoRa RF frequency in kHz. See above for notes about regulatory limits on this +value. + +The antenna and RF matching components on a particular LoRa device may only +support a particular frequency range. Consult the manufacturer's documentation. + +#### `sf` - Spreading Factor +Type: `int` + +Spreading Factor, numeric value only. Higher spreading factors allow reception +of weaker signals but have slower data rate. + +The supported range of SF values varies depending on the modem chipset: + +| Spreading Factor | Supported SX126x | Supported SX127x | +|------------------|------------------|-----------------------| +| 5 | Yes | **No** | +| 6 | **Yes** [*] | **Yes** [*] | +| 7 | Yes | Yes | +| 8 | Yes | Yes | +| 9 | Yes | Yes | +| 10 | Yes | Yes, except SX1277[^] | +| 11 | Yes | Yes, except SX1277[^] | +| 12 | Yes | Yes, except SX2177[^] | + +[*] SF6 is not compatible between SX126x and SX127x chipsets. + +[^] SX1276, SX1278 and SX1279 all support SF6-SF12. SX1277 only supports +SF6-SF9. This limitation is not checked by the driver. + +#### `bw` - Bandwidth +Type: `int` or `str` + +Default: 125 + +Bandwidth value in kHz. Must be exactly one of these LoRa bandwidth values: + +* 7.8 +* 10.4 +* 15.6 +* 20.8 +* 31.25 +* 41.7 +* 62.5 +* 125 +* 250 +* 500 + +Higher bandwidth transmits data faster and reduces peak spectral density when +transmitting, but is more susceptible to interference. + +IF setting bandwidth below 62.5kHz then Semtech recommends using a hardware TCXO +as the modem clock source, not a cheaper crystal. Consult the modem datasheet +and your hardware maker's reference for more information and to determine which +clock source your LoRa modem hardware is using. + +For non-integer bandwidth values, it's recommended to always set this parameter +as a `str` (i.e. `"15.6"`) not a numeric `float`. + +#### `coding_rate` - FEC Coding Rate +Type: `int` + +Default: 5 + +Forward Error Correction (FEC) coding rate is expressed as a ratio, `4/N`. The +value passed in the configuration is `N`: + +| Value | Error coding rate | +|-------|-------------------| +| 5 | 4/5 | +| 6 | 4/6 | +| 7 | 4/7 | +| 8 | 4/8 | + +Setting a higher value makes transmission slower but increases the chance of +receiving successfully in a noisy environment + +In explicit header mode (the default), `coding_rate` only needs to be set by the +transmitter and the receiver will automatically choose the correct rate when +receiving based on the received header. In implicit header mode (see +`implicit_header`), this value must be set the same on both transmitter and +receiver. + +#### `tx_ant` - TX Antenna +Supported: *SX127x only*. + +Type: `str`, not case sensitive + +Default: RFO_HF or RFO_LF (low power) + +SX127x modems and STM32WL55 microcontrollers have multiple antenna pins for +different power levels and frequency ranges. The board/module that the LoRa +modem chip is on may have particular antenna connections, or even an RF switch +that needs to be set via a GPIO to connect an antenna pin to a particular output +(see `ant_sw`, below). + +The driver must configure the modem to use the correct pin for a particular +hardware antenna connection before transmitting. When receiving, the modem +chooses the correct pin based on the selected frequency. + +A common symptom of incorrect `tx_ant` setting is an extremely weak RF signal. + +Consult modem datasheet for more details. + +##### SX127x tx_ant + +| Value | RF Transmit Pin | +|-----------------|----------------------------------| +| `"PA_BOOST"` | PA_BOOST pin (high power) | +| Any other value | RFO_HF or RFO_LF pin (low power) | + +Pin "RFO_HF" is automatically used for frequencies above 862MHz, and is not +supported on SX1278. "RFO_LF" is used for frequencies below 862MHz. Consult +datasheet Table 32 "Frequency Bands" for more details. + +##### WL55SubGhzModem tx_ant + +| Value | RF Transmit Pin | +|-----------------|-------------------------| +| `"PA_BOOST"` | RFO_HP pin (high power) | +| Any other value | RFO_LP pin (low power) | + +**NOTE**: Currently the `PA_BOOST` HP antenna output is lower than it should be +on this board, due to an unknown driver bug. + +If setting `tx_ant` value, also set `output_power` at the same time or again +before transmitting. + +#### `output_power` - Transmit output power level +Type: `int` + +Default: Depends on modem + +Nominal TX output power in dBm. The possible range depends on the modem and for +some modems the `tx_ant` configuration. + +| Modem | `tx_ant` value | Range (dBm) | "Optimal" (dBm) | | +|-----------------|----------------------------|-------------------|------------------------|---| +| SX1261 | N/A | -17 to +15 | +10, +14 or +15 [*][^] | | +| SX1262 | N/A | -9 to +22 | +14, +17, +20, +22 [*] | | +| SX127x | "PA_BOOST" | +2 to +17, or +20 | Any | | +| SX127x | RFO_HF or RFO_LF | -4 to +15 | Any | | +| WL55SubGhzModem | "PA_BOOST" | -9 to +22 | +14, +17, +20, +22 [*] | | +| WL55SubGhzModem | Any other value (not None) | -17 to +14 | +10, +14 or +15 [*][^] | | + +Values which are out of range for the modem will be clamped at the +minimum/maximum values shown above. + +Actual radiated TX power for RF regulatory purposes depends on the RF hardware, +antenna, and the rest of the modem configuration. It should be measured and +tuned empirically not determined from this configuration information alone. + +[*] For some modems the datasheet shows "Optimal" Power Amplifier +configuration values for these output power levels. If setting one of these +levels, the optimal settings from the datasheet are applied automatically by the +driver. Therefore it is recommended to use one of these power levels if +possible. + +[^] In the marked configurations +15dBm is only possible with frequency above +400MHz, will be +14dBm otherwise. + +#### `implicit_header` - Implicit/Explicit Header Mode +Type: `bool` + +Default: `False` + +LoRa supports both implicit and explicit header modes. Explicit header mode +(`implicit_header` set to False) is the default. + +`implicit_header` must be set the same on both sender and receiver. + +* In explicit header mode (default), each transmitted LoRa packet has a header + which contains information about the payload length, `coding_rate` value in + use, and whether the payload has a CRC attached (`crc_en`). The receiving + modem decodes and verifies the header and uses the values to receive the + correct length payload and verify the CRC if enabled. +* In implicit header mode (`implicit_header` set to True), this header is not + sent and this information must be already be known and configured by both + sender and receiver. Specifically: + - `crc_en` setting should be set the same on both sender and receiver. + - `coding_rate` setting must match between the sender and receiver. + - Receiver must provide the `rx_length` argument when calling either + `recv()` or `start_recv()`. This length must match the length in bytes + of the payload sent by the sender. + +### `crc_en` - Enable CRCs +Type: `bool` + +Default: `True` + +LoRa packets can have a 16-bit CRC attached to determine if a packet is +received correctly without corruption. + +* In explicit header mode (default), the sender will attach a CRC if + `crc_en` is True. `crc_en` parameter is ignored by the receiver, which + determines if there is a CRC based on the received header and will check it if + so. +* In implicit header mode, the sender will only include a CRC if `crc_en` + is True and the receiver will only check the CRC if `crc_en` is True. + +By default, if CRC checking is enabled on the receiver then the LoRa modem driver +silently drops packets with invalid CRCs. Setting `modem.rx_crc_error = True` +will change this so that packets with failed CRCs are returned to the caller, +with the `crc_error` field set to True (see `RxPacket`, below). + +#### `auto_image_cal` - Automatic Image Calibration +Supported: *SX127x only*. + +Type: `bool` + +Default: `False` + +If set True, enable automatic image re-calibration in the modem if the +temperature changes significantly. This may avoid RF performance issues caused +by frequency drift, etc. Setting this value may lead to dropped packets received +when an automatic calibration event is in progress. + +Consult SX127x datasheet for more information. + +#### `syncword` - Sync Word +Type: `int` + +Default: `0x12` + +LoRa Sync Words are used to differentiate LoRa packets as being for Public or +Private networks. Sync Word must match between sender and receiver. + +For SX127x this value is an 8-bit integer. Supported values 0x12 for Private +Networks (default, most users) and 0x34 for Public Networks (LoRaWAN only). + +For SX126x this value is a 16-bit integer. Supported values 0x1424 for Private + +Networks (default, most users) and 0x3444 for Public Networks. However the +driver will automatically [translate values configured using the 8-bit SX127x +format](https://www.thethingsnetwork.org/forum/t/should-private-lorawan-networks-use-a-different-sync-word/34496/15) +for software compatibility, so setting an 8-bit value is supported on all modems. + +You probably shouldn't change this value from the default, unless connecting to +a LoRaWAN network. + +#### `pa_ramp_us` - PA Ramp Time +Type: `int` + +Default: `40`us + +Power Amplifier ramp up/down time, as expressed in microseconds. + +The exact values supported on each radio are different. Configuring an +unsupported value will cause the driver to choose the next highest value that is +supported for that radio. + +| Value (us) | Supported SX126x | Supported SX127x | +|------------|------------------|------------------| +| 10 | Yes | Yes | +| 12 | No | Yes | +| 15 | No | Yes | +| 20 | Yes | Yes | +| 25 | No | Yes | +| 31 | No | Yes | +| 40 | Yes | Yes | +| 50 | No | Yes | +| 62 | No | Yes | +| 80 | Yes | No | +| 100 | No | Yes | +| 125 | No | Yes | +| 200 | Yes | No | +| 250 | No | Yes | +| 500 | No | Yes | +| 800 | Yes | No | +| 1000 | No | Yes | +| 1700 | Yes | No | +| 2000 | No | Yes | +| 3400 | Yes | Yes | + +#### `preamble_len` - Preamble Length +Type: `int` +Default: `12` + +Length of the preamble sequence, in units of symbols. + +#### `invert_iq_tx`/`invert_iq_rx` - Invert I/Q +Type: `bool` + +Default: Both `False` + +If `invert_iq_tx` or `invert_iq_rx` is set then IQ polarity is inverted in the +radio for either TX or RX, respectively. The receiver's `invert_iq_rx` setting +must match the sender's `invert_iq_tx` setting. + +This is necessary for LoRaWAN where end-devices transmit with inverted IQ +relative to gateways. + +Note: The current SX127x datasheet incorrectly documents the modem register +setting corresponding to `invert_iq_tx`. This driver configures TX polarity +correctly for compatibility with other LoRa modems, most other SX127x drivers, +and LoRaWAN. However, there are some SX127x drivers that follow the datasheet +description, and they will set `invert_iq_tx` opposite to this. + +#### `rx_boost` - Boost receive sensitivity +Type: `bool` + +Default: `False` + +Enable additional receive sensitivity if available. + +* On SX126x, this makes use of the "Rx Boosted gain" option. +* On SX127x, this option is available for HF bands only and sets the LNA boost + register field. + +#### `lna_gain` - Receiver LNA gain +Supported: *SX127x only*. + +Type: `int` or `None` + +Default: `1` + +Adjust the LNA gain level for receiving. Valid values are `None` to enable +Automatic Gain Control, or integer gain levels 1 to 6 where 1 is maximum gain +(default). + +## Sending & Receiving + +### Simple API + +The driver has a "simple" API to easily send and receive LoRa packets. The +API is fully synchronous, meaning the caller is blocked until the LoRa operation +(send or receive) is done. The Simple API doesn't support starting a +send while a receive in progress (or vice versa). It is suitable for simple +applications only. + +For an example that uses the simple API, see `examples/reliable_delivery/sender.py`. + +#### send + +To send (transmit) a LoRa packet using the configured modulation settings: + +```py +def send(self, packet, tx_at_ms=None) +``` + +Example: + +```py +modem.send(b'Hello world') +``` + +* `send()` transmits a LoRa packet with the provided payload bytes, and returns + once transmission is complete. +* The return value is the timestamp when transmission completed, as a + `time.ticks_ms()` result. It will be more accurate if the modem was + initialized to use interrupts. + +For precise timing of sent packets, there is an optional `tx_at_ms` argument +which is a timestamp (as a `time.ticks_ms()` value). If set, the packet will be +sent as close as possible to this timestamp and the function will block until +that time arrives: + +```py +modem.send(b'Hello world', time.ticks_add(time.ticks_ms(), 250)) +``` + +(This allows more precise timing of sent packets, without needing to account for +the length of the packet to be copied to the modem.) + +### receive + +```py +def recv(self, timeout_ms=None, rx_length=0xFF, rx_packet=None) +``` + +Examples: + +```py +with_timeout = modem.recv(2000) + +print(repr(with_timeout)) + +wait_forever = modem.recv() + +print(repr(wait_forever)) +``` + +* `recv()` receives a LoRa packet from the modem. +* Returns None on timeout, or an `RxPacket` instance with the packet on + success. +* Optional arguments: + - `timeout_ms`. Optional, sets a receive timeout in milliseconds. If None + (default value), then the function will block indefinitely until a packet is + received. + - `rx_length`. Necessary to set if `implicit_header` is set to `True` (see + above). This is the length of the packet to receive. Ignored in the default + LoRa explicit header mode, where the received radio header includes the + length. + - `rx_packet`. Optional, this can be an `RxPacket` object previously + received from the modem. If the newly received packet has the same length, + this object is reused and returned to save an allocation. If the newly + received packet has a different length, a new `RxPacket` object is + allocated and returned instead. + +### RxPacket + +`RxPacket` is a class that wraps a `bytearray` holding the LoRa packet payload, +meaning it can be passed anywhere that accepts a buffer object (like `bytes`, +`bytearray`). + +However it also has the following metadata object variables: + +* `ticks_ms` - is a timestamp of `time.ticks_ms()` called at the time the + packet was received. Timestamp will be more accurate if the modem was + initialized to use interrupts. +* `snr` - is the Signal to Noise ratio of the received packet, in units of `dB * + 4`. Higher values indicate better signal. +* `rssi` - is the Received Signal Strength indicator value in units of + dBm. Higher (less negative) values indicate more signal strength. +* `crc_error` - In the default configuration, this value will always be False as + packets with invalid CRCs are dropped. If the `modem.rx_crc_error` flag is set + to True, then a packet with an invalid CRC will be returned with this flag set + to True. + + Note that CRC is only ever checked on receive in particular configurations, + see the `crc_en` configuration item above for an explanation. If CRC is not + checked on receive, and `crc_error` will always be False. + +Example: + +```py +rx = modem.recv(1000) + +if rx: + print(f'Received {len(rx)} byte packet at ' + f'{rx.ticks_ms}ms, with SNR {rx.snr} ' + f'RSSI {rx.rssi} valid_crc {rx.valid_crc}') +``` + +### Asynchronous API + +Not being able to do anything else while waiting for the modem is very +limiting. Async Python is an excellent match for this kind of application! + +To use async Python, first install `lora-async` and then instantiate the async +version of the LoRA modem class. The async versions have the prefix `Async` at +the beginning of the class name. For example: + +```py +import asyncio +from lora import AsyncSX1276 + +def get_async_modem(): + # The LoRa configuration will depend on your board and location, see + # below under "Modem Configuration" for some possible examples. + lora_cfg = { 'freq_khz': SEE_BELOW_FOR_CORRECT_VALUE } + + # To instantiate SPI correctly, see + # https://docs.micropython.org/en/latest/library/machine.SPI.html + spi = SPI(0, baudrate=2000_000) + cs = Pin(9) + + # or AsyncSX1261, AsyncSX1262, AsyncSX1277, AsyncSX1278, SX1279, etc. + return AsyncSX1276(spi, cs, + dio0=Pin(10), # Optional, recommended + dio1=Pin(11), # Optional, recommended + reset=Pin(13), # Optional, recommended + lora_cfg=lora_cfg) + +modem = get_async_modem() + +async def recv_coro(): + rx = await modem.recv(2000) + if rx: + print(f'Received: {rx}') + else: + print('Timeout!') + + +async def send_coro(): + counter = 0 + while True: + await modem.send(f'Hello world #{counter}'.encode()) + print('Sent!') + await asyncio.sleep(5) + counter += 1 + +async def init(): + await asyncio.gather( + asyncio.create_task(send_coro()), + asyncio.create_task(recv_coro()) + ) + +asyncio.run(init()) +``` + +For a more complete example, see `examples/reliable_delivery/sender_async.py`. + +* The `modem.recv()` and `modem.send()` coroutines take the same + arguments as the synchronous class' functions `recv()` and `send()`, + as documented above. +* However, because these are async coroutines it's possible for other async + tasks to execute while they are blocked waiting for modem operations. +* It is possible to await the `send()` coroutine while a `recv()` + is in progress. The receive will automatically resume once the modem finishes + sending. Send always has priority over receive. +* However, at most one task should be awaiting each of receive and send. For + example, it's not possible for two tasks to `await modem.send()` at the + same time. + +#### Async Continuous Receive + +An additional API provides a Python async iterator that will continuously +receive packets from the modem: + +```py +async def keep_receiving(): + async for packet in am.recv_continuous(): + print(f'Received: {packet}') +``` + +For a more complete example, see `examples/reliable_delivery/receiver_async.py`. + +Receiving will continue and the iterator will yield packets unless another task +calls `modem.stop()` or `modem.standby()` (see below for a description of these +functions). + +Same as the async `recv()` API, it's possible for another task to send while +this iterator is in use. + +## Low-Level API + +This API allows other code to execute while waiting for LoRa operations, without +using asyncio coroutines. + +This is a traditional asynchronous-style API that requires manual management of +modem timing, interrupts, packet timeouts, etc. It's very easy to write +spaghetti code with this API. If asyncio is available on your board, the async +Python API is probably an easier choice to get the same functionality with less +complicated code. + +However, if you absolutely need maximum control over the modem and the rest of +your board then this may be the API for you! + +### Receiving + +```py +will_irq = modem.start_recv(timeout_ms=1000, continuous=False) + +rx = True +while rx is True: + if will_irq: + # Add code to sleep and wait for an IRQ, + # if necessary call modem.irq_triggered() to verify + # that the modem IRQ was actually triggered. + pass + rx = modem.poll_recv() + + # Do anything else you need the application to do + +if rx: # isinstance(rx, lora.RxPacket) + print(f'Received: {rx}') +else: # rx is False + print('Timed out') +``` + +For an example that uses the low-level receive API for continuous receive, see +`examples/reliable_delivery/receiver.py`. + +The steps to receive packet(s) with the low-level API are: + +1. Call `modem.start_recv(timeout_ms=None, continuous=False, rx_length=0xFF)`. + + - `timeout_ms` is an optional timeout in milliseconds, same as the Simple API + recv(). + - Set `continuous=True` for the modem to continuously receive and not go into + standby after the first packet is received. If setting `continuous` to + `True`, `timeout_ms` must be `None`. + - `rx_length` is an optional argument, only used when LoRa implicit headers + are configured. See the Simple API description above for details. + + The return value of this function is truthy if interrupts will be used for + the receive, falsey otherwise. +2. If interrupts are being used, wait for an interrupt to occur. Steps may include + configuring the modem interrupt pins as wake sources and putting the host + into a light sleep mode. See the general description of "Interrupts", below. + + Alternatively, if `timeout_ms` was set then caller can wait for at least the + timeout period before checking if the modem received anything or timed out. + + It is also possible to simply call `poll_recv()` in a loop, but doing + this too frequently may significantly degrade the RF receive performance + depending on the hardware. + +3. Call `modem.poll_recv()`. This function checks the receive state and + returns a value indicating the current state: + + - `True` if the modem is still receiving and the caller should call this + function again in the future. This can be caused by any of: + + * Modem is still waiting in 'single' mode (`continuous=False`) to receive a + packet or time out. + * Modem is in continuous receive mode so will always be receiving. + * The modem is actually sending right now, but the driver will resume + receiving after the send completes. + * The modem received a packet with an invalid CRC (and `modem.rx_crc_error + = False`). The driver has just now discarded it and resumed the modem + receive operation. + + - `False` if the modem is not currently receiving. This can be caused by any + of: + + * No receive has been started. + * A single receive has timed out. + * The receive was aborted. See the `standby()` and `sleep()` functions + below. + + - An instance of the `RxPacket` class. This means the modem has received this + packet since the last call to `poll_recv()`. Whether or not the modem is + still receiving after this depends on whether the receive was started in + `continuous` mode or not.) + +4. If `poll_recv()` returned `True`, go back to step 2 and wait for the next + opportunity to call `poll_recv()`. (Note that it's necessary to test using + `is True` to distinguish between `True` and a new packet.) + +It is possible to also send packets while receiving and looping between +steps 2 and 4. The driver will automatically suspend receiving and resume it +again once sending is done. It's OK to call either the Simple API +`send()` function or the low-level send API (see below) in order to do +this. + +The purpose of the low-level API is to allow code to perform other unrelated +functions during steps 2 and 3. It's still recommended to call +`modem.poll_recv()` as soon as possible after a modem interrupt has +occurred, especially in continuous receive mode when multiple packets may be +received rapidly. + +To cancel a receive in progress, call `modem.standby()` or `modem.sleep()`, see +below for descriptions of these functions. + +*Important*: None of the MicroPython lora driver is thread-safe. It's OK for +different MicroPython threads to manage send and receive, but the caller is +responsible for adding locking so that different threads are not calling any +modem APIs concurrently. Async MicroPython may provide a cleaner and simpler +choice for this kind of firmware architecture. + +### Sending + +The low-level API for sending is similar to the low-level API for receiving: + +1. Call `modem.prepare_send(payload)` with the packet payload. This will put + the modem into standby (pausing receive if necessary), configure the modem + registers, and copy the payload into the modem FIFO buffer. +2. Call `modem.start_send(packet)` to actually start sending. + + Sending is split into these two steps to allow accurate send + timing. `prepare_send()` may take a variable amount of time to copy data + to the modem, configure registers, etc. Then `start_send()` only performs + the minimum fixed duration operation to start sending, so transmit + should start very soon after this function is called. + + The return value of `start_send()` function is truthy if an interrupt is + enabled to signal the send completing, falsey otherwise. + + Not calling both `prepare_send()` or `start_send()` in order, or + calling any other modem functions between `prepare_send()` and + `start_send()`, is not supported and will result in incorrect behaviour. + +3. Wait for the send to complete. This is possible in any of three + different ways: + - If interrupts are enabled, wait for an interrupt to occur. Steps may include + configuring the modem interrupt pins as wake sources and putting the host + into a light sleep mode. See the general description of "Interrupts", below. + - Calculate the packet "time on air" by calling + `modem.get_time_on_air_us(len(packet))` and wait at least this long. + - Call `modem.poll_send()` in a loop (see next step) until it confirms + the send has completed. +4. Call `modem.poll_send()` to check transmission state, and to + automatically resume a receive operation if one was suspended by + `prepare_send()`. The result of this function is one of: + + - `True` if a send is in progress and the caller + should call again. + + - `False` if no send is in progress. + + - An `int` value. This is returned the first time `poll_send()` is + called after a send ended. The value is the `time.ticks_ms()` + timestamp of the time that the send completed. If interrupts are + enabled, this is the time the "send done" ISR executed. Otherwise, it + will be the time that `poll_send()` was just called. + + Note that `modem.poll_send()` returns an `int` only one time per + successful transmission. Any subsequent calls will return `False` as there is + no longer a send in progress. + + To abort a send in progress, call `modem.standby()` or `modem.sleep()`, + see the descriptions of these functions below. Subsequent calls to + `poll_send()` will return `False`. +5. If `poll_send()` returned `True`, repeat steps 3 through 5. + +*Important*: Unless a transmission is aborted, `poll_send()` **MUST be +called** at least once after `start_send()` and should be repeatedly called +until it returns a value other than `True`. `poll_send()` can also be called +after a send is aborted, but this is optional. If `poll_send()` is not +called correctly then the driver's internal state will not correctly update and +no subsequent receive will be able to start. + +It's also possible to mix the simple `send()` API with the low-level receive +API, if this is more convenient for your application. + +### Interrupts + +If interrupt pins are in use then it's important for a programmer using the +low-level API to handle interrupts correctly. + +It's only possible to rely on interrupts if the correct hardware interrupt lines +are configured. Consult the modem reference datasheet, or check if the value of +`start_recv()` or `start_send()` is truthy, in order to know if hardware +interrupts can be used. Otherwise, the modem must be polled to know when an +operation has completed. + +There are two kinds of interrupts: + +* A hardware interrupt (set in the driver by `Pin.irq()`) will be triggered on + the rising edge of a modem interrupt line (DIO0, DIO1, etc). The driver will + attempt to configure these for `RX Done`, `RX Timeout` and `TX Done` events if + possible and applicable for the modem operation, and will handle them. + + It's possible for the programmer to configure these pins as hardware wake sources + and put the board into a low-power sleep mode, to be woken when the modem + finishes its operation. +* A "soft" interrupt is triggered by the driver if an operation is aborted (see + `standby()` and `sleep()`, below), or if a receive operation "soft times + out". A receive "soft times out" if a receive is paused by a send + operation and after the send operation completes then the timeout period + for the receive has already elapsed. In these cases, the driver's radio ISR + routine is called but no hardware interrupt occurs. + +To detect if a modem interrupt has occurred, the programmer can use any of the +following different approaches: + +* Port-specific functions to determine a hardware wakeup cause. Note that this + can only detect hardware interrupts. +* Call the `modem.irq_triggered()` function. This is a lightweight function that + returns True if the modem ISR has been executed since the last time a send + or receive started. It is cleared when `poll_recv()` or `poll_send()` + is called after an interrupt, or when a new operation is started. The idea is + to use this as a lightweight "should I call `poll_recv()` or + `poll_send()` now?" check function if there's no easy way to determine + which interrupt has woken the board up. +* Implement a custom interrupt callback function and call + `modem.set_irq_callback()` to install it. The function will be called if a + hardware interrupt occurs, possibly in hard interrupt context. Refer to the + documentation about [writing interrupt handlers][isr_rules] for more + information. It may also be called if the driver triggers a soft interrupt. + The `lora-async` modem classes install their own callback here, so it's not + possible to mix this approach with the provided asynchronous API. +* Call `modem.poll_recv()` or `modem.poll_send()`. This takes more time + and uses more power as it reads the modem IRQ status directly from the modem + via SPI, but it also give the most definite result. + +As a "belts and braces" protection against unknown driver bugs or modem bugs, +it's best practice to not rely on an interrupt occurring and to also include +some logic that periodically times out and polls the modem state "just in case". + +## Other Functions + +### CRC Error Counter + +Modem objects have a variable `modem.crc_errors` which starts at `0` and +is incremented by one each time a received CRC error or packet header error is +detected by the modem. The programmer can read this value to know the current CRC +error count, and also write it (for example, to clear it periodically by setting +to `0`). + +For an alternative method to know about CRC errors when they occur, set +`modem.rx_crc_error = True` (see `crc_en`, above, for more details.) + +### Modem Standby + +Calling `modem.standby()` puts the modem immediately into standby mode. In the +case of SX1261 and SX1262, the 32MHz oscillator is started. + +Any current send or receive operations are immediately aborted. The +implications of this depends on the API in use: + +* The simple API does not support calling `standby()` while a receive or + send is in progress. +* The async API handles this situation automatically. Any blocked `send()` + or `recv()` async coroutine will return None. The `recv_continuous()` + iterator will stop iterating. +* The low-level API relies on the programmer to handle this case. When the modem + goes to standby, a "soft interrupt" occurs that will trigger the radio ISR and + any related callback, but this is not a hardware interrupt so may not wake the + CPU if the programmer has put it back to sleep. Any subsequent calls to + `poll_recv()` or `poll_send()` will both return `(False, None)` as no + operation is in progress. The programmer needs to ensure that any code that is + blocking waiting for an interrupt has the chance to wake up and call + `poll_recv()` and/or `poll_send()` to detect that the operation(s) have + been aborted. + +### Modem Sleep + +Calling `modem.sleep()` puts the modem into a low power sleep mode with +configuration retention. The modem will automatically wake the next time an +operation is started, or can be woken manually by calling +`modem.standby()`. Waking the modem may take some time, consult the modem +datasheet for details. + +As with `standby()`, any current send or receive operations are immediately +aborted. The implications of this are the same as listed for standby, above. + +### Check if modem is idle + +The `modem.is_idle()` function will return True unless the modem is currently +sending or receiving. + +### Packet length calculations + +Calling `modem.get_time_on_air_us(plen)` will return the "on air time" in +microseconds for a packet of length `plen`, according to the current modem +configuration. This can be used to synchronise modem operations, choose +timeouts, or predict when a send will complete. + +Unlike the other modem API functions, this function doesn't interact with +hardware at all so it can be safely called concurrently with other modem APIs. + +## Antenna switch object + +The modem constructors have an optional `ant_sw` parameter which allows passing +in an antenna switch object to be called by the driver. This allows +automatically configuring some GPIOs or other hardware settings each time the +modem changes between TX and RX modes, and goes idle. + +The argument should be an object which implements three functions: `tx(tx_arg)`, +`rx()`, and `idle()`. For example: + +```py +class MyAntennaSwitch: + def tx(self, tx_arg): + ant_sw_gpio(1) # Set GPIO high + + def rx(self): + ant_sw_gpio(0) # Set GPIO low + + def idle(self): + pass +``` + +* `tx()` is called a short time before the modem starts sending. +* `rx()` is called a short time before the modem starts receiving. +* `idle()` is called at some point after each send or receive completes, and + may be called multiple times. + +The meaning of `tx_arg` depends on the modem: + +* For SX127x it is `True` if the `PA_BOOST` `tx_ant` setting is in use (see + above), and `False` otherwise. +* For SX1262 it is `True` (indicating High Power mode). +* For SX1261 it is `False` (indicating Low Power mode). +* For WL55SubGhzModem it is `True` if the `PA_BOOST` `tx_ant` setting is in use (see above), and `False` otherwise. + +This parameter can be ignored if it's already known what modem and antenna is being used. + +### WL55SubGhzModem ant_sw + +When instantiating the `WL55SubGhzModem` and `AsyncWL55SubGHzModem` classes, the +default `ant_sw` parameter is not `None`. Instead, the default will instantiate +an object of type `lora.NucleoWL55RFConfig`. This implements the antenna switch +connections for the ST NUCLEO-WL55 development board (as connected to GPIO pins +C4, C5 and C3). See ST document [UM2592][ST-UM2592-p27] (PDF) Figure 18 for details. + +When using these modem classes (only), to disable any automatic antenna +switching behaviour it's necessary to explicitly set `ant_sw=None`. + +## Troubleshooting + +Some common errors and their causes: + +### RuntimeError: BUSY timeout + +The SX1261/2 drivers will raise this exception if the modem's TCXO fails to +provide the necessary clock signal when starting a transmit or receive +operation, or moving into "standby" mode. + +Sometimes, this means the constructor parameter `dio3_tcxo_millivolts` (see above) +must be set as the SX126x chip DIO3 output pin is the power source for the TCXO +connected to the modem. Often this parameter should be set to `3300` (3.3V) but +it may be another value, consult the documentation for your LoRa modem module. + +[isr_rules]: https://docs.micropython.org/en/latest/reference/isr_rules.html +[ST-UM2592-p27]: https://www.st.com/resource/en/user_manual/dm00622917-stm32wl-nucleo64-board-mb1389-stmicroelectronics.pdf#page=27 diff --git a/micropython/lora/examples/reliable_delivery/README.md b/micropython/lora/examples/reliable_delivery/README.md new file mode 100644 index 000000000..878b0f8a5 --- /dev/null +++ b/micropython/lora/examples/reliable_delivery/README.md @@ -0,0 +1,93 @@ +# LoRa Reliable Delivery Example + +This example shows a basic custom protocol for reliable one way communication +from low-power remote devices to a central base device: + +- A single "receiver" device, running on mains power, listens continuously for + messages from one or more "sender" devices. Messages are payloads inside LoRa packets, + with some additional framing and address in the LoRa packet payload. +- "Sender" devices are remote sensor nodes, possibly battery powered. These wake + up periodically, read some data from a sensor, and send it in a message to the receiver. +- Messages are transmitted "reliably" with some custom header information, + meaning the receiver will acknowledge it received each message and the sender + will retry sending if it doesn't receive the acknowledgement. + +## Source Files + +* `lora_rd_settings.py` contains some common settings that are imported by + sender and receiver. These settings will need to be modified for the correct + frequency and other settings, before running the examples. +* `receiver.py` and `receiver_async.py` contain a synchronous (low-level API) + and asynchronous (iterator API) implementation of the same receiver program, + respectively. These two programs should work the same, they are intended show + different ways the driver can be used. +* `sender.py` and `sender_async.py` contain a synchronous (simple API) and + asynchronous (async API) implementation of the same sender program, + respectively. Because the standard async API resembles the Simple API, these + implementations are *very* similar. The two programs should work the same, + they are intended to show different ways the driver can be used. + +## Running the examples + +One way to run this example interactively: + +1. Install or "freeze in" the necessary lora modem driver package (`lora-sx127x` + or `lora-sx126x`) and optionally the `lora-async` package if using the async + examples (see main lora `README.md` in the above directory for details). +2. Edit the `lora_rd_settings.py` file to set the frequency and other protocol + settings for your region and hardware (see main lora `README.md`). +3. Edit the program you plan to run and fill in the `get_modem()` function with + the correct modem type, pin assignments, etc. for your board (see top-level + README). Note the `get_modem()` function should use the existing `lora_cfg` + variable, which holds the settings imported from `lora_rd_settings.py`. +4. Change to this directory in a terminal. +5. Run `mpremote mount . exec receiver.py` on one board and `mpremote mount + . exec sender.py` on another (or swap in `receiver_async.py` and/or + `sender_async.py` as desired). + +Consult the [mpremote +documentation](https://docs.micropython.org/en/latest/reference/mpremote.html) +for an explanation of these commands and the options needed to run two copies of +`mpremote` on different serial ports at the same time. + +## Automatic Performance Tuning + +- When sending an ACK, the receiver includes the RSSI of the received + packet. Senders will automatically modify their output_power to minimize the + power consumption required to reach the receiver. Similarly, if no ACK is + received then they will increase their output power and also re-run Image + calibration in order to maximize RX performance. + +## Message payloads + +Messages are LoRa packets, set up as follows: + +LoRA implicit header mode, CRCs enabled. + +* Each remote device has a unique sixteen-bit ID (range 00x0000 to 0xFFFE). ID + 0xFFFF is reserved for the single receiver device. +* An eight-bit message counter is used to identify duplicate messages + +* Data message format is: + - Sender ID (two bytes, little endian) + - Counter byte (incremented on each new message, not incremented on retry). + - Message length (1 byte) + - Message (variable length) + - Checksum byte (sum of all proceeding bytes in message, modulo 256). The LoRa + packet has its own 16-bit CRC, this is included as an additional way to + disambiguate other LoRa packets that might appear the same. + +* After receiving a valid data message, the receiver device should send + an acknowledgement message 25ms after the modem receive completed. + + Acknowledgement message format: + - 0xFFFF (receiver station ID as two bytes) + - Sender's Device ID from received message (two bytes, little endian) + - Counter byte from received message + - Checksum byte from received message + - RSSI value as received by radio (one signed byte) + +* If the remote device doesn't receive a packet with the acknowledgement + message, it retries up to a configurable number of times (default 4) with a + basic exponential backoff formula. + diff --git a/micropython/lora/examples/reliable_delivery/lora_rd_settings.py b/micropython/lora/examples/reliable_delivery/lora_rd_settings.py new file mode 100644 index 000000000..bbf03da5d --- /dev/null +++ b/micropython/lora/examples/reliable_delivery/lora_rd_settings.py @@ -0,0 +1,38 @@ +# MicroPython lora reliable_delivery example - common protocol settings +# MIT license; Copyright (c) 2023 Angus Gratton + +# +###### +# To be able to be able to communicate, most of these settings need to match on both radios. +# Consult the example README for more information about how to use the example. +###### + +# LoRa protocol configuration +# +# Currently configured for relatively slow & low bandwidth settings, which +# gives more link budget and possible range. +# +# These settings should match on receiver. +# +# Check the README and local regulations to know what configuration settings +# are available. +lora_cfg = { + "freq_khz": 916000, + "sf": 10, + "bw": "62.5", # kHz + "coding_rate": 8, + "preamble_len": 12, + "output_power": 10, # dBm +} + +# Single receiver has a fixed 16-bit ID value (senders each have a unique value). +RECEIVER_ID = 0xFFFF + +# Length of an ACK message in bytes. +ACK_LENGTH = 7 + +# Send the ACK this many milliseconds after receiving a valid message +# +# This can be quite a bit lower (25ms or so) if wakeup times are short +# and _DEBUG is turned off on the modems (logging to UART delays everything). +ACK_DELAY_MS = 100 diff --git a/micropython/lora/examples/reliable_delivery/receiver.py b/micropython/lora/examples/reliable_delivery/receiver.py new file mode 100644 index 000000000..2ab4231db --- /dev/null +++ b/micropython/lora/examples/reliable_delivery/receiver.py @@ -0,0 +1,163 @@ +# MicroPython lora reliable_delivery example - synchronous receiver program +# MIT license; Copyright (c) 2023 Angus Gratton +import struct +import time +import machine +from machine import SPI, Pin +from micropython import const +from lora import RxPacket + +from lora_rd_settings import RECEIVER_ID, ACK_LENGTH, ACK_DELAY_MS, lora_cfg + +# Change _DEBUG to const(True) to get some additional debugging output +# about timing, RSSI, etc. +# +# For a lot more debugging detail, go to the modem driver and set _DEBUG there to const(True) +_DEBUG = const(False) + +# Keep track of the last counter value we got from each known sender +# this allows us to tell if packets are being lost +last_counters = {} + + +def get_modem(): + # from lora import SX1276 + # return SX1276( + # spi=SPI(1, baudrate=2000_000, polarity=0, phase=0, + # miso=Pin(19), mosi=Pin(27), sck=Pin(5)), + # cs=Pin(18), + # dio0=Pin(26), + # dio1=Pin(35), + # reset=Pin(14), + # lora_cfg=lora_cfg, + # ) + raise NotImplementedError("Replace this function with one that returns a lora modem instance") + + +def main(): + print("Initializing...") + modem = get_modem() + + print("Main loop started") + receiver = Receiver(modem) + + while True: + # With wait=True, this function blocks until something is received and always + # returns non-None + sender_id, data = receiver.recv(wait=True) + + # Do something with the data! + print(f"Received {data} from {sender_id:#x}") + + +class Receiver: + def __init__(self, modem): + self.modem = modem + self.last_counters = {} # Track the last counter value we got from each sender ID + self.rx_packet = None # Reuse RxPacket object when possible, save allocation + self.ack_buffer = bytearray(ACK_LENGTH) # reuse the same buffer for ACK packets + self.skipped_packets = 0 # Counter of skipped packets + + modem.calibrate() + + # Start receiving immediately. We expect the modem to receive continuously + self.will_irq = modem.start_recv(continuous=True) + print("Modem initialized and started receive...") + + def recv(self, wait=True): + # Receive a packet from the sender, including sending an ACK. + # + # Returns a tuple of the 16-bit sender id and the sensor data payload. + # + # This function should be called very frequently from the main loop (at + # least every ACK_DELAY_MS milliseconds), to avoid not sending ACKs in time. + # + # If 'wait' argument is True (default), the function blocks indefinitely + # until a packet is received. If False then it will return None + # if no packet is available. + # + # Note that because we called start_recv(continuous=True), the modem + # will keep receiving on its own - even if when we call send() to + # send an ACK. + while True: + rx = self.modem.poll_recv(rx_packet=self.rx_packet) + + if isinstance(rx, RxPacket): # value will be True or an RxPacket instance + decoded = self._handle_rx(rx) + if decoded: + return decoded # valid LoRa packet and valid for this application + + if not wait: + return None + + # Otherwise, wait for an IRQ (or have a short sleep) and then poll recv again + # (receiver is not a low power node, so don't bother with sleep modes.) + if self.will_irq: + while not self.modem.irq_triggered(): + machine.idle() + else: + time.sleep_ms(1) + + def _handle_rx(self, rx): + # Internal function to handle a received packet and either send an ACK + # and return the sender and the payload, or return None if packet + # payload is invalid or a duplicate. + + if len(rx) < 5: # 4 byte header plus 1 byte checksum + print("Invalid packet length") + return None + + sender_id, counter, data_len = struct.unpack(" {tx_done}ms took {tx_time}ms expected {expected}") + + # Check if the data we received is fresh or stale + if sender_id not in self.last_counters: + print(f"New device id {sender_id:#x}") + elif self.last_counters[sender_id] == counter: + print(f"Duplicate packet received from {sender_id:#x}") + return None + elif counter != 1: + # If the counter from this sender has gone up by more than 1 since + # last time we got a packet, we know there is some packet loss. + # + # (ignore the case where the new counter is 1, as this probably + # means a reset.) + delta = (counter - 1 - self.last_counters[sender_id]) & 0xFF + if delta: + print(f"Skipped/lost {delta} packets from {sender_id:#x}") + self.skipped_packets += delta + + self.last_counters[sender_id] = counter + return sender_id, rx[4:-1] + + +if __name__ == "__main__": + main() diff --git a/micropython/lora/examples/reliable_delivery/receiver_async.py b/micropython/lora/examples/reliable_delivery/receiver_async.py new file mode 100644 index 000000000..72a456db8 --- /dev/null +++ b/micropython/lora/examples/reliable_delivery/receiver_async.py @@ -0,0 +1,121 @@ +# MicroPython lora reliable_delivery example - asynchronous receiver program +# MIT license; Copyright (c) 2023 Angus Gratton +import struct +import time +import asyncio +from machine import SPI, Pin +from micropython import const + +from lora_rd_settings import RECEIVER_ID, ACK_LENGTH, ACK_DELAY_MS, lora_cfg + +# Change _DEBUG to const(True) to get some additional debugging output +# about timing, RSSI, etc. +# +# For a lot more debugging detail, go to the modem driver and set _DEBUG there to const(True) +_DEBUG = const(False) + +# Keep track of the last counter value we got from each known sender +# this allows us to tell if packets are being lost +last_counters = {} + + +def get_async_modem(): + # from lora import AsyncSX1276 + # return AsyncSX1276( + # spi=SPI(1, baudrate=2000_000, polarity=0, phase=0, + # miso=Pin(19), mosi=Pin(27), sck=Pin(5)), + # cs=Pin(18), + # dio0=Pin(26), + # dio1=Pin(35), + # reset=Pin(14), + # lora_cfg=lora_cfg, + # ) + raise NotImplementedError("Replace this function with one that returns a lora modem instance") + + +def main(): + # Initializing the modem. + # + + print("Initializing...") + modem = get_async_modem() + asyncio.run(recv_continuous(modem, rx_callback)) + + +async def rx_callback(sender_id, data): + # Do something with the data! + print(f"Received {data} from {sender_id:#x}") + + +async def recv_continuous(modem, callback): + # Async task which receives packets from the AsyncModem recv_continuous() + # iterator, checks if they are valid, and send back an ACK if needed. + # + # On each successful message, we await callback() to allow the application + # to do something with the data. Callback args are sender_id (as int) and the bytes + # of the message payload. + + last_counters = {} # Track the last counter value we got from each sender ID + ack_buffer = bytearray(ACK_LENGTH) # reuse the same buffer for ACK packets + skipped_packets = 0 # Counter of skipped packets + + modem.calibrate() + + async for rx in modem.recv_continuous(): + # Filter 'rx' packet to determine if it's valid for our application + if len(rx) < 5: # 4 byte header plus 1 byte checksum + print("Invalid packet length") + continue + + sender_id, counter, data_len = struct.unpack(" {tx_done}ms took {tx_time}ms expected {expected}") + + # Check if the data we received is fresh or stale + if sender_id not in last_counters: + print(f"New device id {sender_id:#x}") + elif last_counters[sender_id] == counter: + print(f"Duplicate packet received from {sender_id:#x}") + continue + elif counter != 1: + # If the counter from this sender has gone up by more than 1 since + # last time we got a packet, we know there is some packet loss. + # + # (ignore the case where the new counter is 1, as this probably + # means a reset.) + delta = (counter - 1 - last_counters[sender_id]) & 0xFF + if delta: + print(f"Skipped/lost {delta} packets from {sender_id:#x}") + skipped_packets += delta + + last_counters[sender_id] = counter + await callback(sender_id, rx[4:-1]) + + +if __name__ == "__main__": + main() diff --git a/micropython/lora/examples/reliable_delivery/sender.py b/micropython/lora/examples/reliable_delivery/sender.py new file mode 100644 index 000000000..957e9d824 --- /dev/null +++ b/micropython/lora/examples/reliable_delivery/sender.py @@ -0,0 +1,213 @@ +# MicroPython lora reliable_delivery example - synchronous sender program +# MIT license; Copyright (c) 2023 Angus Gratton +import machine +from machine import SPI, Pin +import random +import struct +import time + +from lora_rd_settings import RECEIVER_ID, ACK_LENGTH, ACK_DELAY_MS, lora_cfg + +SLEEP_BETWEEN_MS = 5000 # Main loop should sleep this long between sending data to the receiver + +MAX_RETRIES = 4 # Retry each message this often if no ACK is received + +# Initial retry is after this long. Increases by 1.25x each subsequent retry. +BASE_RETRY_TIMEOUT_MS = 1000 + +# Add random jitter to each retry period, up to this long. Useful to prevent two +# devices ending up in sync. +RETRY_JITTER_MS = 1500 + +# If reported RSSI value is lower than this, increase +# output power 1dBm +RSSI_WEAK_THRESH = -110 + +# If reported RSSI value is higher than this, decrease +# output power 1dBm +RSSI_STRONG_THRESH = -70 + +# IMPORTANT: Set this to the maximum output power in dBm that is permitted in +# your regulatory environment. +OUTPUT_MAX_DBM = 15 +OUTPUT_MIN_DBM = -20 + + +def get_modem(): + # from lora import SX1276 + # return SX1276( + # spi=SPI(1, baudrate=2000_000, polarity=0, phase=0, + # miso=Pin(19), mosi=Pin(27), sck=Pin(5)), + # cs=Pin(18), + # dio0=Pin(26), + # dio1=Pin(35), + # reset=Pin(14), + # lora_cfg=lora_cfg, + # ) + raise NotImplementedError("Replace this function with one that returns a lora modem instance") + + +def main(): + modem = get_modem() + + # Unique ID of this sender, 16-bit number. This method of generating an ID is pretty crummy, + # if using this in a real application then probably better to store these in the filesystem or + # something like that + DEVICE_ID = sum(b for b in machine.unique_id()) & 0xFFFF + + sender = Sender(modem, DEVICE_ID) + while True: + sensor_data = get_sensor_data() + sender.send(sensor_data) + + # Sleep until the next time we should read the sensor data and send it to + # the receiver. + # + # The goal for the device firmware is to spend most of its time in the lowest + # available sleep state, to save power. + # + # Note that if the sensor(s) in a real program generates events, these can be + # hooked to interrupts and used to wake Micropython up to send data, + # instead. + modem.sleep() + time.sleep_ms(SLEEP_BETWEEN_MS) # TODO see if this can be machine.lightsleep() + + +def get_sensor_data(): + # Return a bytes object with the latest sensor data to send to the receiver. + # + # As this is just an example, we send a dummy payload which is just a string + # containing our ticks_ms() timestamp. + # + # In a real application the sensor data should usually be binary data and + # not a string, to save transmission size. + return f"Hello, ticks_ms={time.ticks_ms()}".encode() + + +class Sender: + def __init__(self, modem, device_id): + self.modem = modem + self.device_id = device_id + self.counter = 0 + self.output_power = lora_cfg["output_power"] # start with common settings power level + self.rx_ack = None # reuse the ack message object when we can + + print(f"Sender initialized with ID {device_id:#x}") + random.seed(device_id) + self.adjust_output_power(0) # set the initial value within MIN/MAX + + modem.calibrate() + + def send(self, sensor_data, adjust_output_power=True): + # Send a packet of sensor data to the receiver reliably. + # + # Returns True if data was successfully sent and ACKed, False otherwise. + # + # If adjust_output_power==True then increase or decrease output power + # according to the RSSI reported in the ACK packet. + self.counter = (self.counter + 1) & 0xFF + + # Prepare the simple payload with header and checksum + # See README for a summary of the simple data message format + payload = bytearray(len(sensor_data) + 5) + struct.pack_into(" RSSI_STRONG_THRESH: + self.adjust_output_power(-1) + elif rssi < RSSI_WEAK_THRESH: + self.adjust_output_power(1) + + return True + + # Otherwise, prepare to sleep briefly and then retry + next_try_at = time.ticks_add(sent_at, timeout) + sleep_time = time.ticks_diff(next_try_at, time.ticks_ms()) + random.randrange( + RETRY_JITTER_MS + ) + if sleep_time > 0: + self.modem.sleep() + time.sleep_ms(sleep_time) # TODO: see if this can be machine.lightsleep + + # add 25% timeout for next iteration + timeout = (timeout * 5) // 4 + + print(f"Failed, no ACK after {MAX_RETRIES} retries.") + if adjust_output_power: + self.adjust_output_power(2) + self.modem.calibrate_image() # try and improve the RX sensitivity for next time + return False + + def _ack_is_valid(self, maybe_ack, csum): + # Private function to verify if the RxPacket held in 'maybe_ack' is a valid ACK for the + # current device_id and counter value, and provided csum value. + # + # If it is, returns the reported RSSI value from the packet. + # If not, returns None + if (not maybe_ack) or len(maybe_ack) != ACK_LENGTH: + return None + + base_id, ack_id, ack_counter, ack_csum, rssi = struct.unpack(" RSSI_STRONG_THRESH: + self.adjust_output_power(-1) + elif rssi < RSSI_WEAK_THRESH: + self.adjust_output_power(1) + + return True + + # Otherwise, prepare to sleep briefly and then retry + next_try_at = time.ticks_add(sent_at, timeout) + sleep_time = time.ticks_diff(next_try_at, time.ticks_ms()) + random.randrange( + RETRY_JITTER_MS + ) + if sleep_time > 0: + self.modem.sleep() + await asyncio.sleep_ms(sleep_time) + + # add 25% timeout for next iteration + timeout = (timeout * 5) // 4 + + print(f"Failed, no ACK after {MAX_RETRIES} retries.") + if adjust_output_power: + self.adjust_output_power(2) + self.modem.calibrate_image() # try and improve the RX sensitivity for next time + return False + + def _ack_is_valid(self, maybe_ack, csum): + # Private function to verify if the RxPacket held in 'maybe_ack' is a valid ACK for the + # current device_id and counter value, and provided csum value. + # + # If it is, returns the reported RSSI value from the packet. + # If not, returns None + if (not maybe_ack) or len(maybe_ack) != ACK_LENGTH: + return None + + base_id, ack_id, ack_counter, ack_csum, rssi = struct.unpack(" 1 + ): + # This check exists to determine that the SPI settings and modem + # selection are correct. Otherwise it's possible for the driver to + # run for quite some time before it detects an invalid response. + raise RuntimeError("Invalid initial status {}.".format(status)) + + if dio2_rf_sw: + self._cmd("BB", _CMD_SET_DIO2_AS_RF_SWITCH_CTRL, 1) + + if dio3_tcxo_millivolts: + # Enable TCXO power via DIO3, if enabled + # + # timeout register is set in units of 15.625us each, use integer math + # to calculate and round up: + timeout = (dio3_tcxo_start_time_us * 1000 + 15624) // 15625 + if timeout < 0 or timeout > 1 << 24: + raise ValueError("{} out of range".format("dio3_tcxo_start_time_us")) + if dio3_tcxo_millivolts < 1600 or dio3_tcxo_millivolts > 3300: + raise ValueError("{} out of range".format("dio3_tcxo_millivolts")) + dv = dio3_tcxo_millivolts // 100 # 16 to 33 + tcxo_trim_lookup = ( + 16, + 17, + 18, + 22, + 24, + 27, + 30, + 33, + ) # DS Table 13-35 + while dv not in tcxo_trim_lookup: + dv -= 1 + reg_tcxo_trim = tcxo_trim_lookup.index(dv) + + self._cmd(">BI", _CMD_SET_DIO3_AS_TCXO_CTRL, (reg_tcxo_trim << 24) + timeout) + time.sleep_ms(15) + # As per DS 13.3.6 SetDIO3AsTCXOCtrl, should expect error + # value 0x20 "XOSC_START_ERR" to be flagged as XOSC has only just + # started now. So clear it. + self._clear_errors() + + self._check_error() + + # If DIO1 is set, mask in just the IRQs that the driver may need to be + # interrupted by. This is important because otherwise an unrelated IRQ + # can trigger the ISR and may not be reset by the driver, leaving DIO1 high. + # + # If DIO1 is not set, all IRQs can stay masked which is the power-on state. + if dio1: + # Note: we set both Irq mask and DIO1 mask to the same value, which is redundant + # (one could be 0xFFFF) but may save a few bytes of bytecode. + self._cmd( + ">BHHHH", + _CMD_CFG_DIO_IRQ, + (_IRQ_RX_DONE | _IRQ_TX_DONE | _IRQ_TIMEOUT), # IRQ mask + (_IRQ_RX_DONE | _IRQ_TX_DONE | _IRQ_TIMEOUT), # DIO1 mask + 0x0, # DIO2Mask, not used + 0x0, # DIO3Mask, not used + ) + dio1.irq(self._radio_isr, Pin.IRQ_RISING) + + self._clear_irq() + + self._cmd("BB", _CMD_SET_PACKET_TYPE, 1) # LoRa + + if lora_cfg: + self.configure(lora_cfg) + + def sleep(self, warm_start=True): + # Put the modem into sleep mode. Driver will wake the modem automatically the next + # time an operation starts, or call standby() to wake it manually. + # + # If the warm_start parameter is False (non-default) then the modem will + # lose all settings on wake. The only way to use this parameter value is + # to destroy this modem object after calling it, and then instantiate a new + # modem object on wake. + # + self._check_error() # check errors before going to sleep because we clear on wake + self.standby() # save some code size, this clears the driver's rx/tx state + self._cmd("BB", _CMD_SET_SLEEP, _flag(1 << 2, warm_start)) + self._sleep = True + + def _standby(self): + # Send the command for standby mode. + # + # **Don't call this function directly, call standby() instead.** + # + # (This private version doesn't update the driver's internal state.) + self._cmd("BB", _CMD_SET_STANDBY, 1) # STDBY_XOSC mode + self._clear_irq() # clear IRQs in case we just cancelled a send or receive + + def is_idle(self): + # Returns True if the modem is idle (either in standby or in sleep). + # + # Note this function can return True in the case where the modem has temporarily gone to + # standby but there's a receive configured in software that will resume receiving the next + # time poll_recv() or poll_send() is called. + if self._sleep: + return True # getting status wakes from sleep + mode, _ = self._get_status() + return mode in (_STATUS_MODE_STANDBY_HSE32, _STATUS_MODE_STANDBY_RC) + + def _wakeup(self): + # Wake the modem from sleep. This is called automatically the first + # time a modem command is sent after sleep() was called to put the modem to + # sleep. + # + # To manually wake the modem without initiating a new operation, call standby(). + self._cs(0) + time.sleep_us(20) + self._cs(1) + self._sleep = False + self._clear_errors() # Clear "XOSC failed to start" which will reappear at this time + self._check_error() # raise an exception if any other error appears + + def _decode_status(self, raw_status, check_errors=True): + # split the raw status, which often has reserved bits set, into the mode value + # and the command status value + mode = (raw_status & _STATUS_MODE_MASK) >> _STATUS_MODE_SHIFT + cmd = (raw_status & _STATUS_CMD_MASK) >> _STATUS_CMD_SHIFT + if check_errors and cmd in (_STATUS_CMD_EXEC_FAIL, _STATUS_CMD_ERROR): + raise RuntimeError("Status {},{} indicates command error".format(mode, cmd)) + return (mode, cmd) + + def _get_status(self): + # Issue the GetStatus command and return the decoded status of (mode + # value, command status) + # + # Due to what appears to be a silicon bug, we send GetIrqStatus here + # instead of GetStatus. It seems that there is some specific sequence + # where sending command GetStatus to the chip immediately after SetRX + # (mode 5) will trip it it into an endless TX (mode 6) for no apparent + # reason! + # + # It doesn't seem to be timing dependent, all that's needed is that + # ordering (and the modem works fine otherwise). + # + # As a workaround we send the GetIrqStatus command and read an extra two + # bytes that are then ignored... + res = self._cmd("B", _CMD_GET_IRQ_STATUS, n_read=3)[0] + return self._decode_status(res) + + def _check_error(self): + # Raise a RuntimeError if the radio has reported an error state. + # + # Return the decoded status, otherwise. + res = self._cmd("B", _CMD_GET_ERROR, n_read=3) + status = self._decode_status(res[0], False) + op_error = (res[1] << 8) + res[2] + if op_error != 0: + raise RuntimeError("Internal radio Status {} OpError {:#x}".format(status, op_error)) + self._decode_status(res[0]) # raise an exception here if status shows an error + return status + + def _clear_errors(self): + # Clear any errors flagged in the modem + self._cmd(">BH", _CMD_CLR_ERRORS, 0) + + def _clear_irq(self, clear_bits=0xFFFF): + # Clear IRQs flagged in the modem + # + # By default, clears all IRQ bits. Otherwise, argument is the mask of bits to clear. + self._cmd(">BH", _CMD_CLR_IRQ_STATUS, clear_bits) + self._last_irq = None + + def _set_tx_ant(self, tx_ant): + # Only STM32WL55 allows switching tx_ant from LP to HP + raise ConfigError("tx_ant") + + def _symbol_offsets(self): + # Called from BaseModem.get_time_on_air_us(). + # + # This function provides a way to implement the different SF5 and SF6 in SX126x, + # by returning two offsets: one for the overall number of symbols, and one for the + # number of bits used to calculate the symbol length of the payload. + return (2, -8) if self._sf in (5, 6) else (0, 0) + + def configure(self, lora_cfg): + if self._rx is not False: + raise RuntimeError("Receiving") + + if "preamble_len" in lora_cfg: + self._preamble_len = lora_cfg["preamble_len"] + + self._invert_iq = [ + lora_cfg.get("invert_iq_rx", self._invert_iq[0]), + lora_cfg.get("invert_iq_tx", self._invert_iq[1]), + self._invert_iq[2], + ] + + if "freq_khz" in lora_cfg: + self._rf_freq_hz = int(lora_cfg["freq_khz"] * 1000) + rffreq = ( + self._rf_freq_hz << 25 + ) // 32_000_000 # RF-PLL frequency = 32e^6 * RFFreq / 2^25 + if not rffreq: + raise ConfigError("freq_khz") # set to a value too low + self._cmd(">BI", _CMD_SET_RF_FREQUENCY, rffreq) + + if "syncword" in lora_cfg: + syncword = lora_cfg["syncword"] + if syncword < 0x100: + # "Translation from SX127x to SX126x : 0xYZ -> 0xY4Z4 : + # if you do not set the two 4 you might lose sensitivity" + # see + # https://www.thethingsnetwork.org/forum/t/should-private-lorawan-networks-use-a-different-sync-word/34496/15 + syncword = 0x0404 + ((syncword & 0x0F) << 4) + ((syncword & 0xF0) << 8) + self._cmd(">BHH", _CMD_WRITE_REGISTER, _REG_LSYNCRH, syncword) + + if not self._configured or any( + key in lora_cfg for key in ("output_power", "pa_ramp_us", "tx_ant") + ): + pa_config_args, self._output_power = self._get_pa_tx_params( + lora_cfg.get("output_power", self._output_power), lora_cfg.get("tx_ant", None) + ) + self._cmd("BBBBB", _CMD_SET_PA_CONFIG, *pa_config_args) + + if "pa_ramp_us" in lora_cfg: + self._ramp_val = self._get_pa_ramp_val( + lora_cfg, [10, 20, 40, 80, 200, 800, 1700, 3400] + ) + + self._cmd("BBB", _CMD_SET_TX_PARAMS, self._output_power, self._ramp_val) + + if not self._configured or any(key in lora_cfg for key in ("sf", "bw", "coding_rate")): + if "sf" in lora_cfg: + self._sf = lora_cfg["sf"] + if self._sf < _CFG_SF_MIN or self._sf > _CFG_SF_MAX: + raise ConfigError("sf") + + if "bw" in lora_cfg: + self._bw = lora_cfg["bw"] + + if "coding_rate" in lora_cfg: + self._coding_rate = lora_cfg["coding_rate"] + if self._coding_rate < 4 or self._coding_rate > 8: # 4/4 through 4/8, linearly + raise ConfigError("coding_rate") + + bw_val, self._bw_hz = { + "7.8": (0x00, 7800), + "10.4": (0x08, 10400), + "15.6": (0x01, 15600), + "20.8": (0x09, 20800), + "31.25": (0x02, 31250), + "41.7": (0x0A, 41700), + "62.5": (0x03, 62500), + "125": (0x04, 125000), + "250": (0x05, 250000), + "500": (0x06, 500000), + }[str(self._bw)] + + self._cmd( + "BBBBB", + _CMD_SET_MODULATION_PARAMS, + self._sf, + bw_val, + self._coding_rate - 4, # 4/4=0, 4/5=1, etc + self._get_ldr_en(), # Note: BaseModem.get_n_symbols_x4() depends on this logic + ) + + if "rx_boost" in lora_cfg: + # See DS Table 9-3 "Rx Gain Configuration" + self._reg_write(_REG_RX_GAIN, 0x96 if lora_cfg["rx_boost"] else 0x94) + + self._check_error() + self._configured = True + + def _invert_workaround(self, enable): + # Apply workaround for DS 15.4 Optimizing the Inverted IQ Operation + if self._invert_iq[2] != enable: + val = self._reg_read(_REG_IQ_POLARITY_SETUP) + val = (val & ~4) | _flag(4, enable) + self._reg_write(_REG_IQ_POLARITY_SETUP, val) + self._invert_iq[2] = enable + + def _get_irq(self): + # Get currently set IRQ bits. + irq_status = self._cmd("B", _CMD_GET_IRQ_STATUS, n_read=3) + status = self._decode_status(irq_status[0]) + flags = (irq_status[1] << 8) + irq_status[2] + if _DEBUG: + print("Status {} flags {:#x}".format(status, flags)) + return flags + + def calibrate(self): + # Send the Calibrate command to the radio to calibrate RC oscillators, PLL and ADC. + # + # See DS 13.1.12 Calibrate Function + + # calibParam 0xFE means to calibrate all blocks. + self._cmd("BB", _CMD_CALIBRATE, 0xFE) + + time.sleep_us(_CALIBRATE_TYPICAL_TIME_US) + + # a falling edge of BUSY indicates calibration is done + self._wait_not_busy(_CALIBRATE_TIMEOUT_US) + + def calibrate_image(self): + # Send the CalibrateImage command to the modem to improve reception in + # the currently configured frequency band. + # + # See DS 9.2.1 Image Calibration for Specified Frequency Bands + # and 13.1.13 CalibrateImage + + mhz = self._rf_freq_hz // 1_000_000 + if 430 <= mhz <= 440: + args = 0x6B6F + elif 470 <= mhz <= 510: + args = 0x7581 + elif 779 <= mhz <= 787: + args = 0xC1C5 + elif 863 <= mhz <= 870: + args = 0xD7DB + elif 902 <= mhz <= 928: + args = 0xE1E9 + else: + # DS says "Contact your Semtech representative for the other optimal + # calibration settings outside of the given frequency bands" + raise ValueError + + self._cmd(">BH", _CMD_CALIBRATE_IMAGE, args) + + # Can't find anythign in Datasheet about how long image calibration + # takes or exactly how it signals completion. Assuming it will be + # similar to _CMD_CALIBRATE. + self._wait_not_busy(_CALIBRATE_TIMEOUT_US) + + def start_recv(self, timeout_ms=None, continuous=False, rx_length=0xFF): + # Start receiving. + # + # Part of common low-level modem API, see README.md for usage. + super().start_recv(timeout_ms, continuous, rx_length) # sets _rx + + if self._tx: + # Send is in progress and has priority, _check_recv() will start recv + # once send finishes (caller needs to call poll_send() for this to happen.) + if _DEBUG: + print("Delaying receive until send completes") + return self._dio1 + + # Put the modem in a known state. It's possible a different + # receive was in progress, this prevent anything changing while + # we set up the new receive + self._standby() # calling private version to keep driver state as-is + + # Allocate the full FIFO for RX + self._cmd("BBB", _CMD_SET_BUFFER_BASE_ADDRESS, 0xFF, 0x0) + + self._cmd( + ">BHBBBB", + _CMD_SET_PACKET_PARAMS, + self._preamble_len, + self._implicit_header, + rx_length, # PayloadLength, only used in implicit header mode + self._crc_en, # CRCType, only used in implicit header mode + self._invert_iq[0], # InvertIQ + ) + self._invert_workaround(self._invert_iq[0]) + + if continuous: + timeout = _CONTINUOUS_TIMEOUT_VAL + elif timeout_ms is not None: + timeout = max(1, timeout_ms * 64) # units of 15.625us + else: + timeout = 0 # Single receive mode, no timeout + + self._cmd(">BBH", _CMD_SET_RX, timeout >> 16, timeout) # 24 bits + + return self._dio1 + + def poll_recv(self, rx_packet=None): + old_rx = self._rx + rx = super().poll_recv(rx_packet) + + if rx is not True and old_rx is not False and isinstance(old_rx, int): + # Receiving has just stopped, and a timeout was previously set. + # + # Workaround for errata DS 15.3 "Implicit Header Mode Timeout Behaviour", + # which recommends to add the following after "ANY Rx with Timeout active sequence" + self._reg_write(_REG_RTC_CTRL, 0x00) + self._reg_write(_REG_EVT_CLR, self._reg_read(_REG_EVT_CLR) | _REG_EVT_CLR_MASK) + + return rx + + def _rx_flags_success(self, flags): + # Returns True if IRQ flags indicate successful receive. + # Specifically, from the bits in _IRQ_DRIVER_RX_MASK: + # - _IRQ_RX_DONE must be set + # - _IRQ_TIMEOUT must not be set + # - _IRQ_CRC_ERR must not be set + # - _IRQ_HEADER_ERR must not be set + # + # (Note: this is a function because the result for SX1276 depends on + # current config, but the result is constant here.) + return flags & _IRQ_DRIVER_RX_MASK == _IRQ_RX_DONE + + def _read_packet(self, rx_packet, flags): + # Private function to read received packet (RxPacket object) from the + # modem, if there is one. + # + # Called from poll_recv() function, which has already checked the IRQ flags + # and verified a valid receive happened. + + ticks_ms = self._get_last_irq() + + res = self._cmd("B", _CMD_GET_RX_BUFFER_STATUS, n_read=3) + rx_payload_len = res[1] + rx_buffer_ptr = res[2] # should be 0 + + if rx_packet is None or len(rx_packet) != rx_payload_len: + rx_packet = RxPacket(rx_payload_len) + + self._cmd("BB", _CMD_READ_BUFFER, rx_buffer_ptr, n_read=1, read_buf=rx_packet) + + pkt_status = self._cmd("B", _CMD_GET_PACKET_STATUS, n_read=4) + + rx_packet.ticks_ms = ticks_ms + # SNR units are dB * 4 (signed) + rx_packet.rssi, rx_packet.snr = struct.unpack("xBbx", pkt_status) + rx_packet.rssi //= -2 # RSSI, units: dBm + rx_packet.crc_error = (flags & _IRQ_CRC_ERR) != 0 + + return rx_packet + + def prepare_send(self, packet): + # Prepare modem to start sending. Should be followed by a call to start_send() + # + # Part of common low-level modem API, see README.md for usage. + if len(packet) > 255: + raise ConfigError("packet too long") + + # Put the modem in a known state. Any current receive is suspended at this point, + # but calling _check_recv() will resume it later. + self._standby() # calling private version to keep driver state as-is + + self._check_error() + + # Set the board antenna for correct TX mode + if self._ant_sw: + self._ant_sw.tx(self._tx_hp()) + + self._last_irq = None + + self._cmd( + ">BHBBBB", + _CMD_SET_PACKET_PARAMS, + self._preamble_len, + self._implicit_header, + len(packet), + self._crc_en, + self._invert_iq[1], # _invert_iq_tx + ) + self._invert_workaround(self._invert_iq[1]) + + # Allocate the full FIFO for TX + self._cmd("BBB", _CMD_SET_BUFFER_BASE_ADDRESS, 0x0, 0xFF) + self._cmd("BB", _CMD_WRITE_BUFFER, 0x0, write_buf=packet) + + # Workaround for DS 15.1 Modulation Quality with 500 kHZ LoRa Bandwidth + # ... apparently this needs to be done "*before each packet transmission*" + if self._bw_hz == 500_000: + self._reg_write(0x0889, self._reg_read(0x0889) & 0xFB) + else: + self._reg_write(0x0889, self._reg_read(0x0889) | 0x04) + + def start_send(self): + # Actually start a send that was loaded by calling prepare_send(). + # + # This is split into a separate function to allow more precise timing. + # + # The driver doesn't verify the caller has done the right thing here, the + # modem will no doubt do something weird if prepare_send() was not called! + # + # Part of common low-level modem API, see README.md for usage. + + # Currently we don't pass any TX timeout argument to the modem1, + # which the datasheet ominously offers as "security" for the Host MCU if + # the send doesn't start for some reason. + + self._cmd("BBBB", _CMD_SET_TX, 0x0, 0x0, 0x0) + + if _DEBUG: + print("status {}".format(self._get_status())) + self._check_error() + + self._tx = True + + return self._dio1 + + def _wait_not_busy(self, timeout_us): + # Wait until the radio de-asserts the busy line + start = time.ticks_us() + ticks_diff = 0 + while self._busy(): + ticks_diff = time.ticks_diff(time.ticks_us(), start) + if ticks_diff > timeout_us: + raise RuntimeError("BUSY timeout", timeout_us) + time.sleep_us(1) + if _DEBUG and ticks_diff > 105: + # By default, debug log any busy time that takes longer than the + # datasheet-promised Typical 105us (this happens when starting the 32MHz oscillator, + # if it's turned on and off by the modem, and maybe other times.) + print(f"BUSY {ticks_diff}us") + + def _cmd(self, fmt, *write_args, n_read=0, write_buf=None, read_buf=None): + # Execute an SX1262 command + # fmt - Format string suitable for use with struct.pack. First item should be 'B' and + # corresponds to the command opcode. + # write_args - Arguments suitable for struct.pack using fmt. First argument should be a + # command opcode byte. + # + # Optional arguments: + # write_buf - Extra buffer to write from (for FIFO writes). Mutually exclusive with n_read + # or read_buf. + # n_read - Number of result bytes to read back at end + # read_buf - Extra buffer to read into (for FIFO reads) + # + # Returns None if n_read==0, otherwise a memoryview of length n_read which points into a + # shared buffer (buffer will be clobbered on next call to _cmd!) + if self._sleep: + self._wakeup() + + # Ensure "busy" from previously issued command has de-asserted. Usually this will + # have happened well before _cmd() is called again. + self._wait_not_busy(self._busy_timeout) + + # Pack write_args into slice of _buf_view memoryview of correct length + wrlen = struct.calcsize(fmt) + assert n_read + wrlen <= len(self._buf_view) # if this fails, make _buf bigger! + struct.pack_into(fmt, self._buf_view, 0, *write_args) + buf = self._buf_view[: (wrlen + n_read)] + + if _DEBUG: + print(">>> {}".format(buf[:wrlen].hex())) + if write_buf: + print(">>> {}".format(write_buf.hex())) + self._cs(0) + self._spi.write_readinto(buf, buf) + if write_buf: + self._spi.write(write_buf) # Used by _CMD_WRITE_BUFFER only + if read_buf: + self._spi.readinto(read_buf, 0xFF) # Used by _CMD_READ_BUFFER only + self._cs(1) + + if n_read > 0: + res = self._buf_view[wrlen : (wrlen + n_read)] # noqa: E203 + if _DEBUG: + print("<<< {}".format(res.hex())) + return res + + def _reg_read(self, addr): + return self._cmd(">BHB", _CMD_READ_REGISTER, addr, 0, n_read=1)[0] + + def _reg_write(self, addr, val): + return self._cmd(">BHB", _CMD_WRITE_REGISTER, addr, val & 0xFF) + + +class _SX1262(_SX126x): + # Don't construct this directly, construct lora.SX1262 or lora.AsyncSX1262 + def __init__( + self, + spi, + cs, + busy, + dio1=None, + dio2_rf_sw=True, + dio3_tcxo_millivolts=None, + dio3_tcxo_start_time_us=1000, + reset=None, + lora_cfg=None, + ant_sw=None, + ): + super().__init__( + spi, + cs, + busy, + dio1, + dio2_rf_sw, + dio3_tcxo_millivolts, + dio3_tcxo_start_time_us, + reset, + lora_cfg, + ant_sw, + ) + + # Apply workaround for DS 15.2 "Better Resistance of the SX1262 Tx to Antenna Mismatch + self._reg_write(0x8D8, self._reg_read(0x8D8) | 0x1E) + + def _tx_hp(self): + # SX1262 has High Power only (deviceSel==0) + return True + + def _get_pa_tx_params(self, output_power, tx_ant): + # Given an output power level in dB, return a 2-tuple: + # - First item is the 3 arguments for SetPaConfig command + # - Second item is the power level argument value for SetTxParams command. + # + # DS 13.1.14.1 "PA Optimal Settings" gives optimally efficient + # values for output power +22, +20, +17, +14 dBm and "these changes make + # the use of nominal power either sub-optimal or unachievable" (hence it + # recommends setting +22dBm nominal TX Power for all these). + # + # However the modem supports output power as low as -9dBm, and there's + # no explanation in the datasheet of how to best set other output power + # levels. + # + # Semtech's own driver (sx126x.c in LoRaMac-node) only ever executes + # SetPaConfig with the values shown in the datasheet for +22dBm, and + # then executes SetTxParams with power set to the nominal value in + # dBm. + # + # Try for best of both worlds here: If the caller requests an "Optimal" + # value, use the datasheet values. Otherwise set nominal power only as + # per Semtech's driver. + output_power = int(_clamp(output_power, -9, 22)) + + DEFAULT = (0x4, 0x7, 0x0, 0x1) + OPTIMAL = { + 22: (DEFAULT, 22), + 20: ((0x3, 0x5, 0x0, 0x1), 22), + 17: ((0x2, 0x3, 0x0, 0x1), 22), + 14: ((0x2, 0x2, 0x0, 0x1), 22), + } + if output_power in OPTIMAL: + # Datasheet optimal values + return OPTIMAL[output_power] + else: + # Nominal values, as per Semtech driver + return (DEFAULT, output_power & 0xFF) + + +class _SX1261(_SX126x): + # Don't construct this directly, construct lora.SX1261, or lora.AsyncSX1261 + def __init__( + self, + spi, + cs, + busy, + dio1=None, + dio2_rf_sw=True, + dio3_tcxo_millivolts=None, + dio3_tcxo_start_time_us=1000, + reset=None, + lora_cfg=None, + ant_sw=None, + ): + super().__init__( + spi, + cs, + busy, + dio1, + dio2_rf_sw, + dio3_tcxo_millivolts, + dio3_tcxo_start_time_us, + reset, + lora_cfg, + ant_sw, + ) + + def _tx_hp(self): + # SX1261 has Low Power only (deviceSel==1) + return False + + def _get_pa_tx_params(self, output_power, tx_ant): + # Given an output power level in dB, return a 2-tuple: + # - First item is the 3 arguments for SetPaConfig command + # - Second item is the power level argument value for SetTxParams command. + # + # As noted above for SX1262, DS 13.1.14.1 "PA Optimal Settings" + # gives optimally efficient values for output power +15, +14, +10 dBm + # but nothing specific to the other power levels (down to -17dBm). + # + # Therefore do the same as for SX1262 to set optimal values if known, nominal otherwise. + output_power = _clamp(int(output_power), -17, 15) + + DEFAULT = (0x4, 0x0, 0x1, 0x1) + OPTIMAL = { + 15: ((0x06, 0x0, 0x1, 0x1), 14), + 14: (DEFAULT, 14), + 10: ((0x1, 0x0, 0x1, 0x1), 13), + } + + if output_power == 15 and self._rf_freq_hz < 400_000_000: + # DS 13.1.14.1 has Note that PaDutyCycle is limited to 0x4 below 400MHz, + # so disallow the 15dBm optimal setting. + output_power = 14 + + if output_power in OPTIMAL: + # Datasheet optimal values + return OPTIMAL[output_power] + else: + # Nominal values, as per Semtech driver + return (DEFAULT, output_power & 0xFF) + + +# Define the actual modem classes that use the SyncModem & AsyncModem "mixin-like" classes +# to create sync and async variants. + +try: + from .sync_modem import SyncModem + + class SX1261(_SX1261, SyncModem): + pass + + class SX1262(_SX1262, SyncModem): + pass + +except ImportError: + pass + +try: + from .async_modem import AsyncModem + + class AsyncSX1261(_SX1261, AsyncModem): + pass + + class AsyncSX1262(_SX1262, AsyncModem): + pass + +except ImportError: + pass diff --git a/micropython/lora/lora-sx126x/manifest.py b/micropython/lora/lora-sx126x/manifest.py new file mode 100644 index 000000000..76fa91d8d --- /dev/null +++ b/micropython/lora/lora-sx126x/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.5") +require("lora") +package("lora") diff --git a/micropython/lora/lora-sx127x/lora/sx127x.py b/micropython/lora/lora-sx127x/lora/sx127x.py new file mode 100644 index 000000000..9faa79a4d --- /dev/null +++ b/micropython/lora/lora-sx127x/lora/sx127x.py @@ -0,0 +1,889 @@ +# MicroPython LoRa SX127x driver +# MIT license; Copyright (c) 2023 Angus Gratton +# +# LoRa is a registered trademark or service mark of Semtech Corporation or its affiliates. +# +# In comments, abbreviation "DS" = Semtech SX1276/77/78/79 Datasheet rev 7 (May 2020) +from micropython import const +from .modem import BaseModem, ConfigError, RxPacket, _clamp, _flag +from machine import Pin +import struct +import time + +# Set _DEBUG to const(True) to print all register reads and writes, and current register values +# even when an update isn't needed. Plus a few additional pieces of information. +_DEBUG = const(False) + +_WRITE_REG_BIT = const(1 << 7) + +# Registers and fields as bytecode-zerocost constants +# +# Where possible names are direct from DS section 4.4 +# (This means some names are slightly inconsistent, as per datasheet...) + +_REG_FIFO = const(0x00) + +_REG_OPMODE = const(0x01) + +_OPMODE_LONGRANGEMODE_LORA = const(1 << 7) +_OPMODE_LONGRANGEMODE_FSK_OOK = const(0) +_OPMODE_MODE_MASK = const(0x7) +_OPMODE_MODE_SLEEP = const(0x0) +_OPMODE_MODE_STDBY = const(0x1) +_OPMODE_MODE_FSTX = const(0x2) # Frequency synthesis (TX) +_OPMODE_MODE_TX = const(0x3) +_OPMODE_MODE_FSRX = const(0x4) # Frequency synthesis (RX) +_OPMODE_MODE_RX_CONTINUOUS = const(0x5) +_OPMODE_MODE_RX_SINGLE = const(0x6) +_OPMODE_MODE_CAD = const(0x7) # Channel Activity Detection + +_REG_FR_MSB = const(0x06) +_REG_FR_MID = const(0x07) +_REG_FR_LSB = const(0x08) + +_REG_PA_CONFIG = const(0x09) + +_PA_CONFIG_PASELECT_PA_BOOST_PIN = const(1 << 7) +_PA_CONFIG_PASELECT_RFO_PIN = const(0x0) +_PA_CONFIG_MAXPOWER_SHIFT = const(0x4) +_PA_CONFIG_MAXPOWER_MASK = const(0x7) +_PA_CONFIG_OUTPUTPOWER_SHIFT = const(0) +_PA_CONFIG_OUTPUTPOWER_MASK = const(0xF) + +_REG_PA_RAMP = const(0x0A) +_PA_RAMP_MASK = const(0x0F) + +_REG_LNA = const(0x0C) + +_LNA_GAIN_MASK = const(0x7) +_LNA_GAIN_SHIFT = const(5) + +_LNA_BOOST_HF_MASK = 0x3 +_LNA_BOOST_HF_SHIFT = 0x0 + +_REG_FIFO_ADDR_PTR = const(0x0D) +_REG_FIFO_TX_BASE_ADDR = const(0x0E) +_REG_FIFO_RX_BASE_ADDR = const(0x0F) +_REG_FIFO_RX_CURRENT_ADDR = const(0x10) + +_REG_IRQ_FLAGS_MASK = const(0x11) +_REG_IRQ_FLAGS = const(0x12) + +# IRQ mask bits are the same as the IRQ flag bits +_IRQ_RX_TIMEOUT = const(1 << 7) +_IRQ_RX_DONE = const(1 << 6) +_IRQ_PAYLOAD_CRC_ERROR = const(1 << 5) +_IRQ_VALID_HEADER = const(1 << 4) +_IRQ_TX_DONE = const(1 << 3) +_IRQ_CAD_DONE = const(1 << 2) +_IRQ_FHSS_CHANGE_CHANNEL = const(1 << 1) +_IRQ_CAD_DETECTED = const(1 << 0) + +_REG_RX_NB_BYTES = const(0x13) +_REG_RX_HEADER_CNT_VALUE_MSB = const(0x14) +_REG_RX_HEADER_CNT_VALUE_LSB = const(0x13) +_REG_RX_PACKET_CNT_VALUE_MSB = const(0x16) +_REG_RX_PACKET_CNT_VALUE_LSB = const(0x17) + +_REG_MODEM_STAT = const(0x18) +_MODEM_STAT_RX_CODING_RATE_MASK = const(0xE) +_MODEM_STAT_RX_CODING_RATE_SHIFT = const(5) +_MODEM_STAT_MODEM_CLEAR = const(1 << 4) +_MODEM_STAT_HEADER_INFO_VALID = const(1 << 3) +_MODEM_STAT_RX_ONGOING = const(1 << 2) +_MODEM_STAT_SIGNAL_SYNC = const(1 << 1) # Signal synchronized +_MODEM_STAT_SIGNAL_DET = const(1 << 0) # Signal detected + +_REG_PKT_SNR_VAL = const(0x19) +_REG_PKT_RSSI_VAL = const(0x1A) +_REG_RSSI_VAL = const(0x1B) + +_REG_HOP_CHANNEL = const(0x1C) +_HOP_CHANNEL_PLL_TIMEOUT = const(1 << 7) +_HOP_CHANNEL_CRC_ON_PAYLOAD = const(1 << 6) +_HOP_CHANNEL_FHSS_PRESENT_CHANNEL_MASK = const(0x1F) + +_REG_MODEM_CONFIG1 = const(0x1D) +_MODEM_CONFIG1_BW_MASK = const(0xF) +_MODEM_CONFIG1_BW_SHIFT = const(4) +_MODEM_CONFIG1_BW7_8 = const(0x0) +_MODEM_CONFIG1_BW10_4 = const(0x1) +_MODEM_CONFIG1_BW15_6 = const(0x2) +_MODEM_CONFIG1_BW20_8 = const(0x3) +_MODEM_CONFIG1_BW31_25 = const(0x4) +_MODEM_CONFIG1_BW41_7 = const(0x5) +_MODEM_CONFIG1_BW62_5 = const(0x6) +_MODEM_CONFIG1_BW125 = const(0x7) +_MODEM_CONFIG1_BW250 = const(0x8) # not supported in lower band (169MHz) +_MODEM_CONFIG1_BW500 = const(0x9) # not supported in lower band (169MHz) +_MODEM_CONFIG1_CODING_RATE_MASK = const(0x7) +_MODEM_CONFIG1_CODING_RATE_SHIFT = const(1) +_MODEM_CONFIG1_CODING_RATE_45 = const(0b001) +_MODEM_CONFIG1_CODING_RATE_46 = const(0b010) +_MODEM_CONFIG1_CODING_RATE_47 = const(0b011) +_MODEM_CONFIG1_CODING_RATE_48 = const(0b100) +_MODEM_CONFIG1_IMPLICIT_HEADER_MODE_ON = const(1 << 0) + +_REG_MODEM_CONFIG2 = const(0x1E) +_MODEM_CONFIG2_SF_MASK = const(0xF) # Spreading Factor +_MODEM_CONFIG2_SF_SHIFT = const(4) +# SF values are integers 6-12 for SF6-SF12, so skipping constants for these +_MODEM_CONFIG2_SF_MIN = const(6) # inclusive +_MODEM_CONFIG2_SF_MAX = const(12) # inclusive + +_MODEM_CONFIG2_TX_CONTINUOUS = const(1 << 3) +_MODEM_CONFIG2_RX_PAYLOAD_CRC_ON = const(1 << 2) +_MODEM_CONFIG2_SYMB_TIMEOUT_MSB_MASK = 0x3 + +_REG_SYMB_TIMEOUT_LSB = const(0x1F) + +_REG_PREAMBLE_LEN_MSB = const(0x20) +_REG_PREAMBLE_LEN_LSB = const(0x21) + +_REG_PAYLOAD_LEN = const(0x22) # Only for implicit header mode & TX +_REG_MAX_PAYLOAD_LEN = const(0x23) + +_REG_HOP_PERIOD = const(0x24) + +_REG_FIFO_TXBYTE_ADDR = const(0x25) + +_REG_MODEM_CONFIG3 = const(0x26) +_MODEM_CONFIG3_AGC_ON = const(1 << 2) +_MODEM_CONFIG3_LOW_DATA_RATE_OPTIMIZE = const(1 << 3) + +_REG_DETECT_OPTIMIZE = const(0x31) +_DETECT_OPTIMIZE_AUTOMATIC_IF_ON = const( + 1 << 7 +) # Bit should be cleared after reset, as per errata +_DETECT_OPTIMIZE_MASK = 0x7 +_DETECT_OPTIMIZE_SF6 = const(0x05) +_DETECT_OPTIMIZE_OTHER = const(0x03) + +# RegInvertIQ is not correctly documented in DS Rev 7 (May 2020). +# +# The correct behaviour for interoperability with other LoRa devices is as +# written here: +# https://github.com/eclipse/upm/blob/master/src/sx1276/sx1276.cxx#L1310 +# +# Same as used in the Semtech mbed driver, here: +# https://github.com/ARMmbed/mbed-semtech-lora-rf-drivers/blob/master/SX1276/SX1276_LoRaRadio.cpp#L778 +# https://github.com/ARMmbed/mbed-semtech-lora-rf-drivers/blob/master/SX1276/registers/sx1276Regs-LoRa.h#L443 +# +# Specifically: +# - The TX bit in _REG_INVERT_IQ is opposite to what's documented in the datasheet +# (0x01 normal, 0x00 inverted) +# - The RX bit in _REG_INVERT_IQ is as documented in the datasheet (0x00 normal, 0x40 inverted) +# - When enabling LoRa mode, the default register value becomes 0x27 (normal RX & TX) +# rather than the documented power-on value of 0x26. +_REG_INVERT_IQ = const(0x33) +_INVERT_IQ_RX = const(1 << 6) +_INVERT_IQ_TX_OFF = const(1 << 0) + +_REG_DETECTION_THRESHOLD = const(0x37) +_DETECTION_THRESHOLD_SF6 = const(0x0C) +_DETECTION_THRESHOLD_OTHER = const(0x0A) # SF7 to SF12 + +_REG_SYNC_WORD = const(0x39) + +_REG_FSKOOK_IMAGE_CAL = const(0x3B) # NOTE: Only accessible in FSK/OOK mode +_IMAGE_CAL_START = const(1 << 6) +_IMAGE_CAL_RUNNING = const(1 << 5) +_IMAGE_CAL_AUTO = const(1 << 7) + +_REG_INVERT_IQ2 = const(0x3B) +_INVERT_IQ2_ON = const(0x19) +_INVERT_IQ2_OFF = const(0x1D) + +_REG_DIO_MAPPING1 = const(0x40) +_DIO0_MAPPING_MASK = const(0x3) +_DIO0_MAPPING_SHIFT = const(6) +_DIO1_MAPPING_MASK = const(0x3) +_DIO1_MAPPING_SHIFT = const(4) +_DIO2_MAPPING_MASK = const(0x3) +_DIO2_MAPPING_SHIFT = const(2) +_DIO3_MAPPING_MASK = const(0x3) +_DIO3_MAPPING_SHIFT = const(0) + +_REG_DIO_MAPPING2 = const(0x41) +_DIO4_MAPPING_MASK = const(0x3) +_DIO4_MAPPING_SHIFT = const(6) +_DIO5_MAPPING_MASK = const(0x3) +_DIO5_MAPPING_SHIFT = const(4) + +_REG_PA_DAC = const(0x4D) +_PA_DAC_DEFAULT_VALUE = const(0x84) # DS 3.4.3 High Power +20 dBm Operation +_PA_DAC_HIGH_POWER_20DBM = const(0x87) + +_REG_VERSION = const(0x42) + +# IRQs the driver masks in when receiving +_IRQ_DRIVER_RX_MASK = const( + _IRQ_RX_DONE | _IRQ_RX_TIMEOUT | _IRQ_VALID_HEADER | _IRQ_PAYLOAD_CRC_ERROR +) + + +class _SX127x(BaseModem): + # Don't instantiate this class directly, instantiate either lora.SX1276, + # lora.SX1277, lora.SX1278, lora.SX1279, or lora.AsyncSX1276, + # lora.AsyncSX1277, lora.AsyncSX1278, lora.AsyncSX1279 as applicable. + + # common IRQ masks used by the base class functions + _IRQ_RX_COMPLETE = _IRQ_RX_DONE | _IRQ_RX_TIMEOUT + _IRQ_TX_COMPLETE = _IRQ_TX_DONE + + def __init__(self, spi, cs, dio0=None, dio1=None, reset=None, lora_cfg=None, ant_sw=None): + super().__init__(ant_sw) + + self._buf1 = bytearray(1) # shared small buffers + self._buf2 = bytearray(2) + self._spi = spi + self._cs = cs + + self._dio0 = dio0 + self._dio1 = dio1 + + cs.init(Pin.OUT, value=1) + + if dio0: + dio0.init(Pin.IN) + dio0.irq(self._radio_isr, trigger=Pin.IRQ_RISING) + if dio1: + dio1.init(Pin.IN) + dio1.irq(self._radio_isr, trigger=Pin.IRQ_RISING) + + # Configuration settings that need to be tracked by the driver + # Note: a number of these are set in the base class constructor + self._pa_boost = False + + if reset: + # If the user supplies a reset pin argument, reset the radio + reset.init(Pin.OUT, value=0) + time.sleep_ms(1) + reset(1) + time.sleep_ms(5) + + version = self._reg_read(_REG_VERSION) + if version != 0x12: + raise RuntimeError("Unexpected silicon version {}".format(version)) + + # wake the radio and enable LoRa mode if it's not already set + self._set_mode(_OPMODE_MODE_STDBY) + + if lora_cfg: + self.configure(lora_cfg) + + def configure(self, lora_cfg): + if self._rx is not False: + raise RuntimeError("Receiving") + + # Set frequency + if "freq_khz" in lora_cfg: + # Assuming F(XOSC)=32MHz (datasheet both implies this value can be different, and + # specifies it shouldn't be different!) + self._rf_freq_hz = int(lora_cfg["freq_khz"] * 1000) + fr_val = self._rf_freq_hz * 16384 // 1000_000 + buf = bytes([fr_val >> 16, (fr_val >> 8) & 0xFF, fr_val & 0xFF]) + self._reg_write(_REG_FR_MSB, buf) + + # Turn on/off automatic image re-calibration if temperature changes. May lead to dropped + # packets if enabled. + if "auto_image_cal" in lora_cfg: + self._set_mode(_OPMODE_MODE_STDBY, False) # Disable LoRa mode to access FSK/OOK + self._reg_update( + _REG_FSKOOK_IMAGE_CAL, + _IMAGE_CAL_AUTO, + _flag(_IMAGE_CAL_AUTO, lora_cfg["auto_image_cal"]), + ) + self._set_mode(_OPMODE_MODE_STDBY) # Switch back to LoRa mode + + # Note: Common pattern below is to generate a new register value and an update_mask, + # and then call self._reg_update(). self._reg_update() is a + # no-op if update_mask==0 (no bits to change). + + # Update _REG_PA_CONFIG + pa_config = 0x0 + update_mask = 0x0 + + # Ref DS 3.4.2 "RF Power Amplifiers" + if "tx_ant" in lora_cfg: + self._pa_boost = lora_cfg["tx_ant"].upper() == "PA_BOOST" + pa_boost_bit = ( + _PA_CONFIG_PASELECT_PA_BOOST_PIN if self._pa_boost else _PA_CONFIG_PASELECT_RFO_PIN + ) + pa_config |= pa_boost_bit + update_mask |= pa_boost_bit + if not self._pa_boost: + # When using RFO, _REG_PA_DAC can keep default value always + # (otherwise, it's set when output_power is set in next block) + self._reg_write(_REG_PA_DAC, _PA_DAC_DEFAULT_VALUE) + + if "output_power" in lora_cfg: + # See DS 3.4.2 RF Power Amplifiers + dbm = int(lora_cfg["output_power"]) + if self._pa_boost: + if dbm >= 20: + output_power = 0x15 # 17dBm setting + pa_dac = _PA_DAC_HIGH_POWER_20DBM + else: + dbm = _clamp(dbm, 2, 17) # +2 to +17dBm only + output_power = dbm - 2 + pa_dac = _PA_DAC_DEFAULT_VALUE + self._reg_write(_REG_PA_DAC, pa_dac) + else: + # In RFO mode, Output Power is computed from two register fields + # - MaxPower and OutputPower. + # + # Do what the Semtech LoraMac-node driver does here, which is to + # set max_power at one extreme or the other (0 or 7) and then + # calculate the output_power setting based on this baseline. + dbm = _clamp(dbm, -4, 15) + if dbm > 0: + # MaxPower to maximum + pa_config |= _PA_CONFIG_MAXPOWER_MASK << _PA_CONFIG_MAXPOWER_SHIFT + + # Pout (dBm) == 10.8dBm + 0.6*maxPower - (15 - register value) + # 10.8+0.6*7 == 15dBm, so pOut = register_value (0 to 15 dBm) + output_power = dbm + else: + # MaxPower field will be set to 0 + + # Pout (dBm) == 10.8dBm - (15 - OutputPower) + # OutputPower == Pout (dBm) + 4.2 + output_power = dbm + 4 # round down to 4.0, to keep using integer math + + pa_config |= output_power << _PA_CONFIG_OUTPUTPOWER_SHIFT + update_mask |= ( + _PA_CONFIG_OUTPUTPOWER_MASK << _PA_CONFIG_OUTPUTPOWER_SHIFT + | _PA_CONFIG_MAXPOWER_MASK << _PA_CONFIG_MAXPOWER_SHIFT + ) + + self._reg_update(_REG_PA_CONFIG, update_mask, pa_config) + + if "pa_ramp_us" in lora_cfg: + # other fields in this register are reserved to 0 or unused + self._reg_write( + _REG_PA_RAMP, + self._get_pa_ramp_val( + lora_cfg, + [10, 12, 15, 20, 25, 31, 40, 50, 62, 100, 125, 250, 500, 1000, 2000, 3400], + ), + ) + + # If a hard reset happened then flags should be cleared already and mask should + # default to fully enabled, but let's be "belts and braces" sure + self._reg_write(_REG_IRQ_FLAGS, 0xFF) + self._reg_write(_REG_IRQ_FLAGS_MASK, 0) # do IRQ masking in software for now + + # Update MODEM_CONFIG1 + modem_config1 = 0x0 + update_mask = 0x0 + if "bw" in lora_cfg: + bw = str(lora_cfg["bw"]) + bw_reg_val, self._bw_hz = { + "7.8": (_MODEM_CONFIG1_BW7_8, 7800), + "10.4": (_MODEM_CONFIG1_BW10_4, 10400), + "15.6": (_MODEM_CONFIG1_BW15_6, 15600), + "20.8": (_MODEM_CONFIG1_BW20_8, 20800), + "31.25": (_MODEM_CONFIG1_BW31_25, 31250), + "41.7": (_MODEM_CONFIG1_BW41_7, 41700), + "62.5": (_MODEM_CONFIG1_BW62_5, 62500), + "125": (_MODEM_CONFIG1_BW125, 125000), + "250": (_MODEM_CONFIG1_BW250, 250000), + "500": (_MODEM_CONFIG1_BW500, 500000), + }[bw] + modem_config1 |= bw_reg_val << _MODEM_CONFIG1_BW_SHIFT + update_mask |= _MODEM_CONFIG1_BW_MASK << _MODEM_CONFIG1_BW_SHIFT + + if "freq_khz" in lora_cfg or "bw" in lora_cfg: + # Workaround for Errata Note 2.1 "Sensitivity Optimization with a 500 kHz bandwidth" + if self._bw_hz == 500000 and 862_000_000 <= self._rf_freq_hz <= 1020_000_000: + self._reg_write(0x36, 0x02) + self._reg_write(0x3A, 0x64) + elif self._bw_hz == 500000 and 410_000_000 <= self._rf_freq_hz <= 525_000_000: + self._reg_write(0x36, 0x02) + self._reg_write(0x3A, 0x7F) + else: + # "For all other combinations of bandiwdth/frequencies, register at address 0x36 + # should be re-set to value 0x03 and the value at address 0x3a will be + # automatically selected by the chip" + self._reg_write(0x36, 0x03) + + if "coding_rate" in lora_cfg: + self._coding_rate = int(lora_cfg["coding_rate"]) + if self._coding_rate < 5 or self._coding_rate > 8: + raise ConfigError("coding_rate") + # _MODEM_CONFIG1_CODING_RATE_45 == value 5 == 1 + modem_config1 |= (self._coding_rate - 4) << _MODEM_CONFIG1_CODING_RATE_SHIFT + update_mask |= _MODEM_CONFIG1_CODING_RATE_MASK << _MODEM_CONFIG1_CODING_RATE_SHIFT + + if "implicit_header" in lora_cfg: + self._implicit_header = lora_cfg["implicit_header"] + modem_config1 |= _flag(_MODEM_CONFIG1_IMPLICIT_HEADER_MODE_ON, self._implicit_header) + update_mask |= _MODEM_CONFIG1_IMPLICIT_HEADER_MODE_ON + + self._reg_update(_REG_MODEM_CONFIG1, update_mask, modem_config1) + + # Update MODEM_CONFIG2, for any fields that changed + modem_config2 = 0 + update_mask = 0 + if "sf" in lora_cfg: + sf = self._sf = int(lora_cfg["sf"]) + + if sf < _MODEM_CONFIG2_SF_MIN or sf > _MODEM_CONFIG2_SF_MAX: + raise ConfigError("sf") + if sf == 6 and not self._implicit_header: + # DS 4.1.12 "Spreading Factor" + raise ConfigError("SF6 requires implicit_header mode") + + # Update these registers when writing 'SF' + self._reg_write( + _REG_DETECTION_THRESHOLD, + _DETECTION_THRESHOLD_SF6 if sf == 6 else _DETECTION_THRESHOLD_OTHER, + ) + # This field has a reserved non-zero field, so do a read-modify-write update + self._reg_update( + _REG_DETECT_OPTIMIZE, + _DETECT_OPTIMIZE_AUTOMATIC_IF_ON | _DETECT_OPTIMIZE_MASK, + _DETECT_OPTIMIZE_SF6 if sf == 6 else _DETECT_OPTIMIZE_OTHER, + ) + + modem_config2 |= sf << _MODEM_CONFIG2_SF_SHIFT + update_mask |= _MODEM_CONFIG2_SF_MASK << _MODEM_CONFIG2_SF_SHIFT + + if "crc_en" in lora_cfg: + self._crc_en = lora_cfg["crc_en"] + # I had to double-check the datasheet about this point: + # 1. In implicit header mode, this bit is used on both RX & TX and + # should be set to get CRC generation on TX and/or checking on RX. + # 2. In explicit header mode, this bit is only used on TX (should CRC + # be added and CRC flag set in header) and ignored on RX (CRC flag + # read from header instead). + modem_config2 |= _flag(_MODEM_CONFIG2_RX_PAYLOAD_CRC_ON, self._crc_en) + update_mask |= _MODEM_CONFIG2_RX_PAYLOAD_CRC_ON + + self._reg_update(_REG_MODEM_CONFIG2, update_mask, modem_config2) + + # Update _REG_INVERT_IQ + # + # See comment about this register's undocumented weirdness at top of + # file above _REG_INVERT_IQ constant. + # + # Note also there is a second register invert_iq2 which may be set differently + # for transmit vs receive, see _set_invert_iq2() for that one. + invert_iq = 0x0 + update_mask = 0x0 + if "invert_iq_rx" in lora_cfg: + self._invert_iq[0] = lora_cfg["invert_iq_rx"] + invert_iq |= _flag(_INVERT_IQ_RX, lora_cfg["invert_iq_rx"]) + update_mask |= _INVERT_IQ_RX + if "invert_iq_tx" in lora_cfg: + self._invert_iq[1] = lora_cfg["invert_iq_tx"] + invert_iq |= _flag(_INVERT_IQ_TX_OFF, not lora_cfg["invert_iq_tx"]) # Inverted + update_mask |= _INVERT_IQ_TX_OFF + self._reg_update(_REG_INVERT_IQ, update_mask, invert_iq) + + if "preamble_len" in lora_cfg: + self._preamble_len = lora_cfg["preamble_len"] + self._reg_write(_REG_PREAMBLE_LEN_MSB, struct.pack(">H", self._preamble_len)) + + # Update MODEM_CONFIG3, for any fields that have changed + modem_config3 = 0 + update_mask = 0 + + if "sf" in lora_cfg or "bw" in lora_cfg: + # Changing either SF or BW means the Low Data Rate Optimization may need to be changed + # + # note: BaseModem.get_n_symbols_x4() assumes this value is set automatically + # as follows. + modem_config3 |= _flag(_MODEM_CONFIG3_LOW_DATA_RATE_OPTIMIZE, self._get_ldr_en()) + update_mask |= _MODEM_CONFIG3_LOW_DATA_RATE_OPTIMIZE + + if "lna_gain" in lora_cfg: + lna_gain = lora_cfg["lna_gain"] + update_mask |= _MODEM_CONFIG3_AGC_ON + if lna_gain is None: # Setting 'None' means 'Auto' + modem_config3 |= _MODEM_CONFIG3_AGC_ON + else: # numeric register value + # Clear the _MODEM_CONFIG3_AGC_ON bit, and write the manual LNA gain level 1-6 + # to the register + self._reg_update( + _REG_LNA, _LNA_GAIN_MASK << _LNA_GAIN_SHIFT, lna_gain << _LNA_GAIN_SHIFT + ) + + if "rx_boost" in lora_cfg: + self._reg_update( + _REG_LNA, + _LNA_BOOST_HF_MASK << _LNA_BOOST_HF_SHIFT, + _flag(0x3, lora_cfg["lna_boost_hf"]), + ) + + self._reg_update(_REG_MODEM_CONFIG3, update_mask, modem_config3) + + if "syncword" in lora_cfg: + self._reg_write(_REG_SYNC_WORD, lora_cfg["syncword"]) + + def _reg_write(self, reg, value): + self._cs(0) + if isinstance(value, int): + self._buf2[0] = reg | _WRITE_REG_BIT + self._buf2[1] = value + self._spi.write(self._buf2) + if _DEBUG: + dbg = hex(value) + else: # value is a buffer + self._buf1[0] = reg | _WRITE_REG_BIT + self._spi.write(self._buf1) + self._spi.write(value) + if _DEBUG: + dbg = value.hex() + self._cs(1) + + if _DEBUG: + print("W {:#x} ==> {}".format(reg, dbg)) + self._reg_read(reg) # log the readback as well + + def _reg_update(self, reg, update_mask, new_value): + # Update register address 'reg' with byte value new_value, as masked by + # bit mask update_mask. Bits not set in update_mask will be kept at + # their pre-existing values in the register. + # + # If update_mask is zero, this function is a no-op and returns None. + # If update_mask is not zero, this function updates 'reg' and returns + # the previous complete value of 'reg' as a result. + # + # Note: this function has no way of detecting a race condition if the + # modem updates any bits in 'reg' that are unset in update_mask, at the + # same time a read/modify/write is occurring. Any such changes are + # overwritten with the original values. + + if not update_mask: # short-circuit if nothing to change + if _DEBUG: + # Log the current value if DEBUG is on + # (Note the compiler will optimize this out otherwise) + self._reg_read(reg) + return + old_value = self._reg_read(reg) + value = ((old_value & ~update_mask) & 0xFF) | (new_value & update_mask) + if old_value != value: + self._reg_write(reg, value) + return old_value + + def _reg_read(self, reg): + # Read and return a single register value at address 'reg' + self._buf2[0] = reg + self._buf2[1] = 0xFF + self._cs(0) + self._spi.write_readinto(self._buf2, self._buf2) + self._cs(1) + if _DEBUG: + print("R {:#x} <== {:#x}".format(reg, self._buf2[1])) + return self._buf2[1] + + def _reg_readinto(self, reg, buf): + # Read and return one or more register values starting at address 'reg', + # into buffer 'buf'. + self._cs(0) + self._spi.readinto(self._buf1, reg) + self._spi.readinto(buf) + if _DEBUG: + print("R {:#x} <== {}".format(reg, buf.hex())) + self._cs(1) + + def _get_mode(self): + # Return the current 'Mode' field in RegOpMode + return self._reg_read(_REG_OPMODE) & _OPMODE_MODE_MASK + + def _set_mode(self, mode, lora_en=True): + # Set the 'Mode' and 'LongRangeMode' fields in RegOpMode + # according to 'mode' and 'lora_en', respectively. + # + # If enabling or disabling LoRa mode, the radio is automatically + # switched into Sleep mode as required and then the requested mode is + # set (if not sleep mode). + # + # Returns the previous value of the RegOpMode register (unmasked). + mask = _OPMODE_LONGRANGEMODE_LORA | _OPMODE_MODE_MASK + lora_val = _flag(_OPMODE_LONGRANGEMODE_LORA, lora_en) + old_value = self._reg_read(_REG_OPMODE) + new_value = (old_value & ~mask) | lora_val | mode + + if lora_val != (old_value & _OPMODE_LONGRANGEMODE_LORA): + # Need to switch into Sleep mode in order to change LongRangeMode flag + self._reg_write(_REG_OPMODE, _OPMODE_MODE_SLEEP | lora_val) + + if new_value != old_value: + self._reg_write(_REG_OPMODE, new_value) + + if _DEBUG: + print( + "Mode {} -> {} ({:#x})".format( + old_value & _OPMODE_MODE_MASK, mode, self._reg_read(_REG_OPMODE) + ) + ) + + return old_value + + def _set_invert_iq2(self, val): + # Set the InvertIQ2 register on/off as needed, unless it is already set to the correct + # level + if self._invert_iq[2] == val: + return # already set to the level we want + self._reg_write(_REG_INVERT_IQ2, _INVERT_IQ2_ON if val else _INVERT_IQ2_OFF) + self._invert_iq[2] = val + + def _standby(self): + # Send the command for standby mode. + # + # **Don't call this function directly, call standby() instead.** + # + # (This private version doesn't update the driver's internal state.) + old_mode = self._set_mode(_OPMODE_MODE_STDBY) & _OPMODE_MODE_MASK + if old_mode not in (_OPMODE_MODE_STDBY, _OPMODE_MODE_SLEEP): + # If we just cancelled sending or receiving, clear any pending IRQs + self._reg_write(_REG_IRQ_FLAGS, 0xFF) + + def sleep(self): + # Put the modem into sleep mode. Modem will wake automatically the next + # time host asks it for something, or call standby() to wake it manually. + self.standby() # save some code size, this clears driver state for us + self._set_mode(_OPMODE_MODE_SLEEP) + + def is_idle(self): + # Returns True if the modem is idle (either in standby or in sleep). + # + # Note this function can return True in the case where the modem has temporarily gone to + # standby, but there's a receive configured in software that will resume receiving the + # next time poll_recv() or poll_send() is called. + return self._get_mode() in (_OPMODE_MODE_STDBY, _OPMODE_MODE_SLEEP) + + def calibrate_image(self): + # Run the modem Image & RSSI calibration process to improve receive performance. + # + # calibration will be run in the HF or LF band automatically, depending on the + # current radio configuration. + # + # See DS 2.1.3.8 Image and RSSI Calibration. Idea to disable TX power + # comes from Semtech's sx1276 driver which does this. + + pa_config = self._reg_update(_REG_PA_CONFIG, 0xFF, 0) # disable TX power + + self._set_mode(_OPMODE_MODE_STDBY, False) # Switch to FSK/OOK mode to expose RegImageCal + + self._reg_update(_REG_FSKOOK_IMAGE_CAL, _IMAGE_CAL_START, _IMAGE_CAL_START) + while self._reg_read(_REG_FSKOOK_IMAGE_CAL) & _IMAGE_CAL_RUNNING: + time.sleep_ms(1) + + self._set_mode(_OPMODE_MODE_STDBY) # restore LoRA mode + + self._reg_write(_REG_PA_CONFIG, pa_config) # restore previous TX power + + def calibrate(self): + # Run a full calibration. + # + # For SX1276, this means just the image & RSSI calibration as no other runtime + # calibration is implemented in the modem. + self.calibrate_image() + + def start_recv(self, timeout_ms=None, continuous=False, rx_length=0xFF): + # Start receiving. + # + # Part of common low-level modem API, see README.md for usage. + super().start_recv(timeout_ms, continuous, rx_length) # sets self._rx + + # will_irq if DIO0 and DIO1 both hooked up, or DIO0 and no timeout + will_irq = self._dio0 and (self._dio1 or timeout_ms is None) + + if self._tx: + # Send is in progress and has priority, _check_recv() will start receive + # once send finishes (caller needs to call poll_send() for this to happen.) + if _DEBUG: + print("Delaying receive until send completes") + return will_irq + + # Put the modem in a known state. It's possible a different + # receive was in progress, this prevent anything changing while + # we set up the new receive + self._standby() # calling private version to keep driver state as-is + + # Update the InvertIQ2 setting for RX + self._set_invert_iq2(self._invert_iq[0]) + + if self._implicit_header: + # Payload length only needs to be set in implicit header mode + self._reg_write(_REG_PAYLOAD_LEN, rx_length) + + if self._dio0: + # Field value is 0, for DIO0 = RXDone + update_mask = _DIO0_MAPPING_MASK << _DIO0_MAPPING_SHIFT + if self._dio1: + # Field value also 0, for DIO1 = RXTimeout + update_mask |= _DIO1_MAPPING_MASK << _DIO1_MAPPING_SHIFT + self._reg_update(_REG_DIO_MAPPING1, update_mask, 0) + + if not continuous: + # Unlike SX1262, SX1276 doesn't have a "single RX no timeout" mode. So we set the + # maximum hardware timeout and resume RX in software if needed. + if timeout_ms is None: + timeout_syms = 1023 + else: + t_sym_us = self._get_t_sym_us() + timeout_syms = (timeout_ms * 1000 + t_sym_us - 1) // t_sym_us # round up + + # if the timeout is too long for the modem, the host will + # automatically resume it in software. If the timeout is too + # short for the modem, round it silently up to the minimum + # timeout. + timeout_syms = _clamp(timeout_syms, 4, 1023) + self._reg_update( + _REG_MODEM_CONFIG2, + _MODEM_CONFIG2_SYMB_TIMEOUT_MSB_MASK, + timeout_syms >> 8, + ) + self._reg_write(_REG_SYMB_TIMEOUT_LSB, timeout_syms & 0xFF) + + # Allocate the full FIFO for RX + self._reg_write(_REG_FIFO_ADDR_PTR, 0) + self._reg_write(_REG_FIFO_RX_BASE_ADDR, 0) + + self._set_mode(_OPMODE_MODE_RX_CONTINUOUS if continuous else _OPMODE_MODE_RX_SINGLE) + + return will_irq + + def _rx_flags_success(self, flags): + # Returns True if IRQ flags indicate successful receive. + # Specifically, from the bits in _IRQ_DRIVER_RX_MASK: + # - _IRQ_RX_DONE must be set + # - _IRQ_RX_TIMEOUT must not be set + # - _IRQ_PAYLOAD_CRC_ERROR must not be set + # - _IRQ_VALID_HEADER must be set if we're using explicit packet mode, ignored otherwise + return flags & _IRQ_DRIVER_RX_MASK == _IRQ_RX_DONE | _flag( + _IRQ_VALID_HEADER, not self._implicit_header + ) + + def _get_irq(self): + return self._reg_read(_REG_IRQ_FLAGS) + + def _clear_irq(self, to_clear=0xFF): + return self._reg_write(_REG_IRQ_FLAGS, to_clear) + + def _read_packet(self, rx_packet, flags): + # Private function to read received packet (RxPacket object) from the + # modem, if there is one. + # + # Called from poll_recv() function, which has already checked the IRQ flags + # and verified a valid receive happened. + + ticks_ms = self._get_last_irq() # IRQ timestamp for the receive + + rx_payload_len = self._reg_read(_REG_RX_NB_BYTES) + + if rx_packet is None or len(rx_packet) != rx_payload_len: + rx_packet = RxPacket(rx_payload_len) + + self._reg_readinto(_REG_FIFO, rx_packet) + + rx_packet.ticks_ms = ticks_ms + # units: dB*4 + rx_packet.snr = self._reg_read(_REG_PKT_SNR_VAL) + if rx_packet.snr & 0x80: # Signed 8-bit integer + # (avoiding using struct here to skip a heap allocation) + rx_packet.snr -= 0x100 + # units: dBm + rx_packet.rssi = self._reg_read(_REG_PKT_RSSI_VAL) - (157 if self._pa_boost else 164) + rx_packet.crc_error = flags & _IRQ_PAYLOAD_CRC_ERROR != 0 + return rx_packet + + def prepare_send(self, packet): + # Prepare modem to start sending. Should be followed by a call to start_send() + # + # Part of common low-level modem API, see README.md for usage. + if len(packet) > 255: + raise ValueError("packet too long") + + # Put the modem in a known state. Any current receive is suspended at this point, + # but calling _check_recv() will resume it later. + self._standby() # calling private version to keep driver state as-is + + if self._ant_sw: + self._ant_sw.tx(self._pa_boost) + + self._last_irq = None + + if self._dio0: + self._reg_update( + _REG_DIO_MAPPING1, + _DIO0_MAPPING_MASK << _DIO0_MAPPING_SHIFT, + 1 << _DIO0_MAPPING_SHIFT, + ) # DIO0 = TXDone + + # Update the InvertIQ2 setting for TX + self._set_invert_iq2(self._invert_iq[1]) + + # Allocate the full FIFO for TX + self._reg_write(_REG_FIFO_ADDR_PTR, 0) + self._reg_write(_REG_FIFO_TX_BASE_ADDR, 0) + + self._reg_write(_REG_PAYLOAD_LEN, len(packet)) + + self._reg_write(_REG_FIFO, packet) + + # clear the TX Done flag in case a previous call left it set + # (won't happen unless poll_send() was not called) + self._reg_write(_REG_IRQ_FLAGS, _IRQ_TX_DONE) + + def start_send(self): + # Actually start a send that was loaded by calling prepare_send(). + # + # This is split into a separate function to allow more precise timing. + # + # The driver doesn't verify the caller has done the right thing here, the + # modem will no doubt do something weird if prepare_send() was not called! + # + # Part of common low-level modem API, see README.md for usage. + self._set_mode(_OPMODE_MODE_TX) + + self._tx = True + + return self._dio0 is not None # will_irq if dio0 is set + + def _irq_flag_tx_done(self): + return _IRQ_TX_DONE + + +# Define the actual modem classes that use the SyncModem & AsyncModem "mixin-like" classes +# to create sync and async variants. + +try: + from .sync_modem import SyncModem + + class SX1276(_SX127x, SyncModem): + pass + + # Implementation note: Currently the classes SX1276, SX1277, SX1278 and + # SX1279 are actually all SX1276. Perhaps in the future some subclasses with + # software enforced limits can be added to this driver, but the differences + # appear very minor: + # + # - SX1276 seems like "baseline" with max freq. + # - SX1277 supports max SF level of 9. + # - SX1278 supports max freq 525MHz, therefore has no RFO_HF and RFI_HF pins. + # - SX1279 supports max freq 960MHz. + # + # There also appears to be no difference in silicon interface or register values to determine + # which model is connected. + SX1277 = SX1278 = SX1279 = SX1276 + +except ImportError: + pass + +try: + from .async_modem import AsyncModem + + class AsyncSX1276(_SX127x, AsyncModem): + pass + + # See comment above about currently identical implementations + AsyncSX1277 = AsyncSX1278 = AsyncSX1279 = AsyncSX1276 + +except ImportError: + pass diff --git a/micropython/lora/lora-sx127x/manifest.py b/micropython/lora/lora-sx127x/manifest.py new file mode 100644 index 000000000..177877091 --- /dev/null +++ b/micropython/lora/lora-sx127x/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.2") +require("lora") +package("lora") diff --git a/micropython/lora/lora-sync/lora/sync_modem.py b/micropython/lora/lora-sync/lora/sync_modem.py new file mode 100644 index 000000000..585ae2cb4 --- /dev/null +++ b/micropython/lora/lora-sync/lora/sync_modem.py @@ -0,0 +1,86 @@ +# MicroPython LoRa synchronous modem driver +# MIT license; Copyright (c) 2023 Angus Gratton +# +# LoRa is a registered trademark or service mark of Semtech Corporation or its affiliates. + +import machine +import time + + +class SyncModem: + # Mixin-like base class that provides synchronous modem send and recv + # functions + # + # + # Don't instantiate this class directly, instantiate one of the 'AsyncXYZ' + # modem classes defined in the lora module. + # + # These are intended for simple applications. They block the caller until + # the modem operation is complete, and don't support interleaving send + # and receive. + + def _after_init(self): + pass # Needed for AsyncModem but not SyncModem + + def send(self, packet, tx_at_ms=None): + # Send the given packet (byte sequence), + # and return once transmission of the packet is complete. + # + # Returns a timestamp (result of time.ticks_ms()) when the packet + # finished sending. + self.prepare_send(packet) + + # If the caller specified a timestamp to start transmission at, wait until + # that time before triggering the send + if tx_at_ms is not None: + time.sleep_ms(max(0, time.ticks_diff(tx_at_ms, time.ticks_ms()))) + + will_irq = self.start_send() # ... and go! + + # sleep for the expected send time before checking if send has ended + time.sleep_ms(self.get_time_on_air_us(len(packet)) // 1000) + + tx = True + while tx is True: + self._sync_wait(will_irq) + tx = self.poll_send() + return tx + + def recv(self, timeout_ms=None, rx_length=0xFF, rx_packet=None): + # Attempt to a receive a single LoRa packet, timeout after timeout_ms milliseconds + # or wait indefinitely if no timeout is supplied (default). + # + # Returns an instance of RxPacket or None if the radio timed out while receiving. + # + # Optional rx_length argument is only used if lora_cfg["implict_header"] == True + # (not the default) and holds the length of the payload to receive. + # + # Optional rx_packet argument can be an existing instance of RxPacket + # which will be reused to save allocations, but only if the received packet + # is the same length as the rx_packet packet. If the length is different, a + # new RxPacket instance is allocated and returned. + will_irq = self.start_recv(timeout_ms, False, rx_length) + rx = True + while rx is True: + self._sync_wait(will_irq) + rx = self.poll_recv(rx_packet) + return rx or None + + def _sync_wait(self, will_irq): + # For synchronous usage, block until an interrupt occurs or we time out + if will_irq: + for n in range(100): + machine.idle() + # machine.idle() wakes up very often, so don't actually return + # unless _radio_isr ran already. The outer for loop is so the + # modem is still polled occasionally to + # avoid the possibility an IRQ was lost somewhere. + # + # None of this is very efficient, power users should either use + # async or call the low-level API manually with better + # port-specific sleep configurations, in order to get the best + # efficiency. + if self.irq_triggered(): + break + else: + time.sleep_ms(1) diff --git a/micropython/lora/lora-sync/manifest.py b/micropython/lora/lora-sync/manifest.py new file mode 100644 index 000000000..1936a50e4 --- /dev/null +++ b/micropython/lora/lora-sync/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.1") +require("lora") +package("lora") diff --git a/micropython/lora/lora/lora/__init__.py b/micropython/lora/lora/lora/__init__.py new file mode 100644 index 000000000..7f8930b8c --- /dev/null +++ b/micropython/lora/lora/lora/__init__.py @@ -0,0 +1,40 @@ +# MicroPython lora module +# MIT license; Copyright (c) 2023 Angus Gratton + +from .modem import RxPacket # noqa: F401 + +ok = False # Flag if at least one modem driver package is installed + +# Various lora "sub-packages" + +try: + from .sx126x import * # noqa: F401 + + ok = True +except ImportError as e: + if "no module named 'lora." not in str(e): + raise + +try: + from .sx127x import * # noqa: F401 + + ok = True +except ImportError as e: + if "no module named 'lora." not in str(e): + raise + +try: + from .stm32wl5 import * # noqa: F401 + + ok = True +except ImportError as e: + if "no module named 'lora." not in str(e): + raise + + +if not ok: + raise ImportError( + "Incomplete lora installation. Need at least one of lora-sync, lora-async and one of lora-sx126x, lora-sx127x" + ) + +del ok diff --git a/micropython/lora/lora/lora/modem.py b/micropython/lora/lora/lora/modem.py new file mode 100644 index 000000000..499712acf --- /dev/null +++ b/micropython/lora/lora/lora/modem.py @@ -0,0 +1,474 @@ +# MicroPython LoRa modem driver base class +# MIT license; Copyright (c) 2023 Angus Gratton +# +# LoRa is a registered trademark or service mark of Semtech Corporation or its affiliates. +import time +from micropython import const, schedule + +# Set to True to get some additional printed debug output. +_DEBUG = const(False) + + +def _clamp(v, vmin, vmax): + # Small utility function to clamp a value 'v' between 'vmin' and 'vmax', inclusive. + return min(max(vmin, v), vmax) + + +def _flag(value, condition): + # Small utility function for returning a bit 'value' or not, based on a + # boolean condition. Can help make expressions to build register values more + # readable. + # + # Note that for value==1, can also rely on int(bool(x)) with one or both + # conversions being implicit, as int(True)==1 and int(False)==0 + # + # There is also (condition and value) but this is (IMO) confusing to read. + return value if condition else 0 + + +class ConfigError(ValueError): + # Raise if there is an error in lora_cfg, saves some duplicated strings + def __init__(self, field): + super().__init__("Invalid lora_cfg {}".format(field)) + + +class BaseModem: + def __init__(self, ant_sw): + self._ant_sw = ant_sw + self._irq_callback = None + + # Common configuration settings that need to be tracked by all modem drivers. + # + # Where modem hardware sets different values after reset, the driver should + # set them back to these defaults (if not provided by the user), so that + # behaviour remains consistent between different modems using the same driver. + self._rf_freq_hz = 0 # Needs to be set via configure() + self._sf = 7 # Spreading factor + self._bw_hz = 125000 # Reset value + self._coding_rate = 5 + self._crc_en = True # use packet CRCs + self._implicit_header = False # implict vs explicit header mode + self._preamble_len = 12 + self._coding_rate = 5 + + # CRC error counter + self.crc_errors = 0 + self.rx_crc_error = False + + # Current state of the modem + + # _rx holds radio recv state: + # + # - False if the radio is not receiving + # - True if the radio is continuously receiving, or performing a single receive with + # no timeout. + # - An int if there is a timeout set, in which case it is the is the receive deadline + # (as a time.ticks_ms() timestamp). + # + # Note that self._rx can be not-False even when the radio hardware is not actually + # receiving, if self._tx is True (send always pauses recv.) + self._rx = False + + # _rx_continuous is True if the modem is in continuous receive mode + # (this value is only valid when self._rx is also True). + self._rx_continuous = False + + # This argument is stored from the parameter of the same name, as set in + # the last call to start_recv() + self._rx_length = None + + # _tx holds radio send state and is simpler, True means sending and + # False means not sending. + self._tx = False + + # timestamp (as time.ticks_ms() result) of last IRQ event + self._last_irq = None + + # values are: + # - lora_cfg["invert_iq_rx"] + # - lora_cfg["invert_iq_tx"] + # - Current modem Invert setting + self._invert_iq = [False, False, False] + + # This hook exists to allow the SyncModem & AsyncModem "mixin-like" + # classes to have some of their own state, without needing to manage the + # fuss of multiple constructor paths. + try: + self._after_init() + except AttributeError: + # If this exception happens here then one of the modem classes without a SyncModem or AsyncModem "mixin-like" class + # has been instantiated. + raise NotImplementedError( + "Don't instantiate this class directly, " + "instantiate a class from the 'lora' package" + ) + + def standby(self): + # Put the modem into standby. Can be used to cancel a continuous recv, + # or cancel a send before it completes. + # + # Calls the private function which actually sets the mode to standby, and then + # clears all the driver's state flags. + # + # Note this is also called before going to sleep(), to save on duplicated code. + self._standby() + self._rx = False + self._tx = False + self._last_irq = None + if self._ant_sw: + self._ant_sw.idle() + self._radio_isr(None) # "soft ISR" + + def _get_t_sym_us(self): + # Return length of a symbol in microseconds + return 1000_000 * (1 << self._sf) // self._bw_hz + + def _get_ldr_en(self): + # Return true if Low Data Rate should be enabled + # + # The calculation in get_n_symbols_x4() relies on this being the same logic applied + # in the modem configuration routines. + return self._get_t_sym_us() >= 16000 + + def _get_pa_ramp_val(self, lora_cfg, supported): + # Return the PA ramp register index from the list of supported PA ramp + # values. If the requested ramp time is supported by the modem, round up + # to the next supported value. + # + # 'supported' is the list of supported ramp times, must be sorted + # already. + us = int(lora_cfg["pa_ramp_us"]) + + # Find the index of the lowest supported ramp time that is longer or the + # same value as 'us' + for i, v in enumerate(supported): + if v >= us: + return i + # The request ramp time is longer than all this modem's supported ramp times + raise ConfigError("pa_ramp_us") + + def _symbol_offsets(self): + # Called from get_time_on_air_us(). + # + # This function provides a way to implement the different SF5 and SF6 in SX126x, + # by returning two offsets: one for the overall number of symbols, and one for the + # number of bits used to calculate the symbol length of the payload. + return (0, 0) + + def get_n_symbols_x4(self, payload_len): + # Get the number of symbols in a packet (Time-on-Air) for the current + # configured modem settings and the provided payload length in bytes. + # + # Result is in units of "symbols times 4" as there is a fractional term + # in the equation, and we want to limit ourselves to integer arithmetic. + # + # References are: + # - SX1261/2 DS 6.1.4 "LoRa Time-on-Air" + # - SX1276 DS 4.1.1 "Time on air" + # + # Note the two datasheets give the same information in different + # ways. SX1261/62 DS is (IMO) clearer, so this function is based on that + # formula. The result is equivalent to the datasheet value "Nsymbol", + # times 4. + # + # Note also there are unit tests for this function in tests/test_time_on_air.py, + # and that it's been optimised a bit for code size (with impact on readability) + + # Account for a minor difference between SX126x and SX127x: they have + # incompatible SF 5 & 6 modes. + # + # In SX126x when using SF5 or SF6, we apply an offset of +2 symbols to + # the overall preamble symbol count (s_o), and an offset of -8 to the + # payload bit length (b_o). + s_o, b_o = self._symbol_offsets() + + # calculate the bit length of the payload + # + # This is the part inside the max(...,0) in the datasheet + bits = ( + # payload_bytes + 8 * payload_len + # N_bit_crc + + (16 if self._crc_en else 0) + # (4 * SF) + - (4 * self._sf) + # +8 for most modes, except SF5/6 on SX126x where b_o == -8 so these two cancel out + + 8 + + b_o + # N_symbol_header + + (0 if self._implicit_header else 20) + ) + bits = max(bits, 0) + + # "Bits per symbol" denominator is either (4 * SF) or (4 * (SF -2)) + # depending on Low Data Rate Optimization + bps = (self._sf - (2 * self._get_ldr_en())) * 4 + + return ( + # Fixed preamble portion (4.25), times 4 + 17 + # Remainder of equation is an integer number of symbols, times 4 + + 4 + * ( + # configured preamble length + self._preamble_len + + + # optional extra preamble symbols (4.25+2=6.25 for SX1262 SF5,SF6) + s_o + + + # 8 symbol constant overhead + 8 + + + # Payload symbol length + # (this is the term "ceil(bits / 4 * SF) * (CR + 4)" in the datasheet + ((bits + bps - 1) // bps) * self._coding_rate + ) + ) + + def get_time_on_air_us(self, payload_len): + # Return the "Time on Air" in microseconds for a particular + # payload length and the current configured modem settings. + return self._get_t_sym_us() * self.get_n_symbols_x4(payload_len) // 4 + + # Modem ISR routines + # + # ISR implementation is relatively simple, just exists to signal an optional + # callback, record a timestamp, and wake up the hardware if + # needed. Application code is expected to call poll_send() or + # poll_recv() as applicable in order to confirm the modem state. + # + # This is a MP hard irq in some configurations. + def _radio_isr(self, _): + self._last_irq = time.ticks_ms() + if self._irq_callback: + self._irq_callback() + if _DEBUG: + print("_radio_isr") + + def irq_triggered(self): + # Returns True if the ISR has executed since the last time a send or a receive + # started + return self._last_irq is not None + + def set_irq_callback(self, callback): + # Set a function to be called from the radio ISR + # + # This is used by the AsyncModem implementation, but can be called in + # other circumstances to implement custom ISR logic. + # + # Note that callback may be called in hard ISR context. + self._irq_callback = callback + + def _get_last_irq(self): + # Return the _last_irq timestamp if set by an ISR, or the + # current time.time_ms() timestamp otherwise. + if self._last_irq is None: + return time.ticks_ms() + return self._last_irq + + # Common parts of receive API + + def start_recv(self, timeout_ms=None, continuous=False, rx_length=0xFF): + # Start receiving. + # + # Part of common low-level modem API, see README.md for usage. + if continuous and timeout_ms is not None: + raise ValueError # these two options are mutually exclusive + + if timeout_ms is not None: + self._rx = time.ticks_add(time.ticks_ms(), timeout_ms) + else: + self._rx = True + + self._rx_continuous = continuous + self._rx_length = rx_length + + if self._ant_sw and not self._tx: + # this is guarded on 'not self._tx' as the subclass will not immediately + # start receiving if a send is in progress. + self._ant_sw.rx() + + def poll_recv(self, rx_packet=None): + # Should be called while a receive is in progress: + # + # Part of common low-level modem API, see README.md for usage. + # + # This function may alter the state of the modem - it will clear + # RX interrupts, it may read out a packet from the FIFO, and it + # may resume receiving if the modem has gone to standby but receive + # should resume. + + if self._rx is False: + # Not actually receiving... + return False + + if self._tx: + # Actually sending, this has to complete before we + # resume receiving, but we'll indicate that we are still receiving. + # + # (It's not harmful to fall through here and check flags anyhow, but + # it is a little wasteful if an interrupt has just triggered + # poll_send() as well.) + return True + + packet = None + + flags = self._get_irq() + + if _DEBUG and flags: + print("RX flags {:#x}".format(flags)) + if flags & self._IRQ_RX_COMPLETE: + # There is a small potential for race conditions here in continuous + # RX mode. If packets are received rapidly and the call to this + # function delayed, then a ValidHeader interrupt (for example) might + # have already set for a second packet which is being received now, + # and clearing it will mark the second packet as invalid. + # + # However it's necessary in continuous mode as interrupt flags don't + # self-clear in the modem otherwise (for example, if a CRC error IRQ + # bit sets then it stays set on the next packet, even if that packet + # has a valid CRC.) + self._clear_irq(flags) + ok = self._rx_flags_success(flags) + if not ok: + # If a non-valid receive happened, increment the CRC error counter + self.crc_errors += 1 + if ok or self.rx_crc_error: + # Successfully received a valid packet (or configured to return all packets) + packet = self._read_packet(rx_packet, flags) + if not self._rx_continuous: + # Done receiving now + self._end_recv() + + # _check_recv() will return True if a receive is ongoing and hasn't timed out, + # and also manages resuming any modem receive if needed + # + # We need to always call check_recv(), but if we received a packet then this is what + # we should return to the caller. + res = self._check_recv() + return packet or res + + def _end_recv(self): + # Utility function to clear the receive state + self._rx = False + if self._ant_sw: + self._ant_sw.idle() + + def _check_recv(self): + # Internal function to automatically call start_recv() + # again if a receive has been interrupted and the host + # needs to start it again. + # + # Return True if modem is still receiving (or sending, but will + # resume receiving after send finishes). + + if not self._rx: + return False # Not receiving, nothing to do + + if not self.is_idle(): + return True # Radio is already sending or receiving + + rx = self._rx + + timeout_ms = None + if isinstance(rx, int): # timeout is set + timeout_ms = time.ticks_diff(rx, time.ticks_ms()) + if timeout_ms <= 0: + # Timed out in software, nothing to resume + self._end_recv() + if _DEBUG: + print("Timed out in software timeout_ms={}".format(timeout_ms)) + schedule( + self._radio_isr, None + ) # "soft irq" to unblock anything waiting on the interrupt event + return False + + if _DEBUG: + print( + "Resuming receive timeout_ms={} continuous={} rx_length={}".format( + timeout_ms, self._rx_continuous, self._rx_length + ) + ) + + self.start_recv(timeout_ms, self._rx_continuous, self._rx_length) + + # restore the previous version of _rx so ticks_ms deadline can't + # slowly creep forward each time this happens + self._rx = rx + + return True + + # Common parts of send API + + def poll_send(self): + # Check the ongoing send state. + # + # Returns one of: + # + # - True if a send is ongoing and the caller + # should call again. + # - False if no send is ongoing. + # - An int value exactly one time per transmission, the first time + # poll_send() is called after a send ends. In this case it + # is the time.ticks_ms() timestamp of the time that the send completed. + # + # Note this function only returns an int value one time (the first time it + # is called after send completes). + # + # Part of common low-level modem API, see README.md for usage. + if not self._tx: + return False + + ticks_ms = self._get_last_irq() + + if not (self._get_irq() & self._IRQ_TX_COMPLETE): + # Not done. If the host and modem get out + # of sync here, or the caller doesn't follow the sequence of + # send operations exactly, then can end up in a situation here + # where the modem has stopped sending and has gone to Standby, + # so _IRQ_TX_DONE is never set. + # + # For now, leaving this for the caller to do correctly. But if it becomes an issue then + # we can call _get_mode() here as well and check the modem is still in a TX mode. + return True + + self._clear_irq() + + self._tx = False + + if self._ant_sw: + self._ant_sw.idle() + + # The modem just finished sending, so start receiving again if needed + self._check_recv() + + return ticks_ms + + +class RxPacket(bytearray): + # A class to hold a packet received from a LoRa modem. + # + # The base class is bytearray, which represents the packet payload, + # allowing RxPacket objects to be passed anywhere that bytearrays are + # accepted. + # + # Some additional properties are set on the object to store metadata about + # the received packet. + def __init__(self, payload, ticks_ms=None, snr=None, rssi=None, valid_crc=True): + super().__init__(payload) + self.ticks_ms = ticks_ms + self.snr = snr + self.rssi = rssi + self.valid_crc = valid_crc + + def __repr__(self): + return "{}({}, {}, {}, {}, {})".format( + "RxPacket", + repr( + bytes(self) + ), # This is a bit wasteful, but gets us b'XYZ' rather than "bytearray(b'XYZ')" + self.ticks_ms, + self.snr, + self.rssi, + self.valid_crc, + ) diff --git a/micropython/lora/lora/manifest.py b/micropython/lora/lora/manifest.py new file mode 100644 index 000000000..586c47c08 --- /dev/null +++ b/micropython/lora/lora/manifest.py @@ -0,0 +1,2 @@ +metadata(version="0.2.0") +package("lora") diff --git a/micropython/lora/tests/test_time_on_air.py b/micropython/lora/tests/test_time_on_air.py new file mode 100644 index 000000000..56fa1ad81 --- /dev/null +++ b/micropython/lora/tests/test_time_on_air.py @@ -0,0 +1,310 @@ +# MicroPython LoRa modem driver time on air tests +# MIT license; Copyright (c) 2023 Angus Gratton +# +# LoRa is a registered trademark or service mark of Semtech Corporation or its affiliates. +# +# ## What is this? +# +# Host tests for the BaseModem.get_time_on_air_us() function. Theses against +# dummy test values produced by the Semtech "SX1261 LoRa Calculator" software, +# as downloaded from +# https://lora-developers.semtech.com/documentation/product-documents/ +# +# The app notes for SX1276 (AN1200.3) suggest a similar calculator exists for that +# modem, but it doesn't appear to be available for download any more. I couldn't find +# an accurate calculator for SX1276, so manually calculated the SF5 & SF6 test cases below +# (other values should be the same as SX1262). +# +# ## Instructions +# +# These tests are intended to be run on a host PC via micropython unix port: +# +# cd /path/to/micropython-lib/micropython/lora +# micropython -m tests.test_time_on_air +# +# Note: Using the working directory shown above is easiest way to ensure 'lora' files are imported. +# +from lora import SX1262, SX1276 + +# Allow time calculations to deviate by up to this much as a ratio +# of the expected value (due to floating point, etc.) +TIME_ERROR_RATIO = 0.00001 # 0.001% + + +def main(): + sx1262 = SX1262(spi=DummySPI(), cs=DummyPin(), busy=DummyPin()) + sx1276 = SX1276(spi=DummySPI(0x12), cs=DummyPin()) + + # Test case format is based on the layout of the Semtech Calculator UI: + # + # (modem_instance, + # (modem settings), + # [ + # ((packet config), (output values)), + # ... + # ], + # ), + # + # where each set of modem settings maps to zero or more packet config / output pairs + # + # - modem instance is sx1262 or sx1276 (SF5 & SF6 are different between these modems) + # - (modem settings) is (sf, bw (in khz), coding_rate, low_datarate_optimize) + # - (packet config) is (preamble_len, payload_len, explicit_header, crc_en) + # - (output values) is (total_symbols_excl, symbol_time in ms, time_on_air in ms) + # + # NOTE: total_symbols_excl is the value shown in the calculator output, + # which doesn't include 8 symbols of constant overhead between preamble and + # header+payload+crc. I think this is a bug in the Semtech calculator(!). + # These 8 symbols are included when the calculator derives the total time on + # air. + # + # NOTE ALSO: The "symbol_time" only depends on the modem settings so is + # repeated each group of test cases, and the "time_on_air" is the previous + # two output values multiplied (after accounting for the 8 symbols noted + # above). This repetition is deliberate to make the cases easier to read + # line-by-line when comparing to the calculator window. + CASES = [ + ( + sx1262, + (12, 500, 5, False), # Calculator defaults when launching calculator + [ + ((8, 1, True, True), (17.25, 8.192, 206.848)), # Calculator defaults + ((12, 64, True, True), (71.25, 8.192, 649.216)), + ((8, 1, True, False), (12.25, 8.192, 165.888)), + ((8, 192, True, True), (172.25, 8.192, 1476.608)), + ((12, 16, False, False), (26.25, 8.192, 280.576)), + ], + ), + ( + sx1262, + (8, 125, 6, False), + [ + ((8, 1, True, True), (18.25, 2.048, 53.760)), + ((8, 2, True, True), (18.25, 2.048, 53.760)), + ((8, 2, True, False), (18.25, 2.048, 53.760)), + ((8, 3, True, True), (24.25, 2.048, 66.048)), + ((8, 3, True, False), (18.25, 2.048, 53.760)), + ((8, 4, True, True), (24.25, 2.048, 66.048)), + ((8, 4, True, False), (18.25, 2.048, 53.760)), + ((8, 5, True, True), (24.25, 2.048, 66.048)), + ((8, 5, True, False), (24.25, 2.048, 66.048)), + ((8, 253, True, True), (396.25, 2.048, 827.904)), + ((8, 253, True, False), (396.25, 2.048, 827.904)), + ((12, 5, False, True), (22.25, 2.048, 61.952)), + ((12, 5, False, False), (22.25, 2.048, 61.952)), + ((12, 10, False, True), (34.25, 2.048, 86.528)), + ((12, 253, False, True), (394.25, 2.048, 823.808)), + ], + ), + # quick check that sx1276 is the same as sx1262 for SF>6 + ( + sx1276, + (8, 125, 6, False), + [ + ((8, 1, True, True), (18.25, 2.048, 53.760)), + ((8, 2, True, True), (18.25, 2.048, 53.760)), + ((12, 5, False, True), (22.25, 2.048, 61.952)), + ((12, 5, False, False), (22.25, 2.048, 61.952)), + ], + ), + # SF5 on SX1262 + ( + sx1262, + (5, 500, 5, False), + [ + ( + (2, 1, True, False), + (13.25, 0.064, 1.360), + ), # Shortest possible LoRa packet? + ((2, 1, True, True), (18.25, 0.064, 1.680)), + ((12, 1, False, False), (18.25, 0.064, 1.680)), + ((12, 253, False, True), (523.25, 0.064, 34.000)), + ], + ), + ( + sx1262, + (5, 125, 8, False), + [ + ((12, 253, False, True), (826.25, 0.256, 213.568)), + ], + ), + # SF5 on SX1276 + # + # Note: SF5 & SF6 settings are different between SX1262 & SX1276. + # + # There's no Semtech official calculator available for SX1276, so the + # symbol length is calculated by copying the formula from the datasheet + # "Time on air" section. Symbol time is the same as SX1262. Then the + # time on air is manually calculated by multiplying the two together. + # + # see the functions sx1276_num_payload and sx1276_num_symbols at end of this module + # for the actual functions used. + ( + sx1276, + (5, 500, 5, False), + [ + ( + (2, 1, True, False), + (19.25 - 8, 0.064, 1.232), + ), # Shortest possible LoRa packet? + ((2, 1, True, True), (24.25 - 8, 0.064, 1.552)), + ((12, 1, False, False), (24.25 - 8, 0.064, 1.552)), + ((12, 253, False, True), (534.25 - 8, 0.064, 34.192)), + ], + ), + ( + sx1276, + (5, 125, 8, False), + [ + ((12, 253, False, True), (840.25 - 8, 0.256, 215.104)), + ], + ), + ( + sx1262, + (12, 7.81, 8, True), # Slowest possible + [ + ((128, 253, True, True), (540.25, 524.456, 287532.907)), + ((1000, 253, True, True), (1412.25, 524.456, 744858.387)), + ], + ), + ( + sx1262, + (11, 10.42, 7, True), + [ + ((25, 16, True, True), (57.25, 196.545, 12824.568)), + ((25, 16, False, False), (50.25, 196.545, 11448.752)), + ], + ), + ] + + tests = 0 + failures = set() + for modem, modem_settings, packets in CASES: + (sf, bw_khz, coding_rate, low_datarate_optimize) = modem_settings + print( + f"Modem config sf={sf} bw={bw_khz}kHz coding_rate=4/{coding_rate} " + + f"low_datarate_optimize={low_datarate_optimize}" + ) + + # We don't call configure() as the Dummy interfaces won't handle it, + # just update the BaseModem fields directly + modem._sf = sf + modem._bw_hz = int(bw_khz * 1000) + modem._coding_rate = coding_rate + + # Low datarate optimize on/off is auto-configured in the current driver, + # check the automatic selection matches the test case from the + # calculator + if modem._get_ldr_en() != low_datarate_optimize: + print( + f" -- ERROR: Test case has low_datarate_optimize={low_datarate_optimize} " + + f"but modem selects {modem._get_ldr_en()}" + ) + failures += 1 + continue # results will not match so don't run any of the packet test cases + + for packet_config, expected_outputs in packets: + preamble_len, payload_len, explicit_header, crc_en = packet_config + print( + f" -- preamble_len={preamble_len} payload_len={payload_len} " + + f"explicit_header={explicit_header} crc_en={crc_en}" + ) + modem._preamble_len = preamble_len + modem._implicit_header = not explicit_header # opposite logic to calculator + modem._crc_en = crc_en + + # Now calculate the symbol length and times and compare with the expected valuesd + ( + expected_symbols, + expected_symbol_time, + expected_time_on_air, + ) = expected_outputs + + print(f" ---- calculator shows total length {expected_symbols}") + expected_symbols += 8 # Account for the calculator bug mentioned in the comment above + + n_symbols = modem.get_n_symbols_x4(payload_len) / 4.0 + symbol_time_us = modem._get_t_sym_us() + time_on_air_us = modem.get_time_on_air_us(payload_len) + + tests += 1 + + if n_symbols == expected_symbols: + print(f" ---- symbols {n_symbols}") + else: + print(f" ---- SYMBOL COUNT ERROR expected {expected_symbols} got {n_symbols}") + failures.add((modem, modem_settings, packet_config)) + + max_error = expected_symbol_time * 1000 * TIME_ERROR_RATIO + if abs(int(expected_symbol_time * 1000) - symbol_time_us) <= max_error: + print(f" ---- symbol time {expected_symbol_time}ms") + else: + print( + f" ---- SYMBOL TIME ERROR expected {expected_symbol_time}ms " + + f"got {symbol_time_us}us" + ) + failures.add((modem, modem_settings, packet_config)) + + max_error = expected_time_on_air * 1000 * TIME_ERROR_RATIO + if abs(int(expected_time_on_air * 1000) - time_on_air_us) <= max_error: + print(f" ---- time on air {expected_time_on_air}ms") + else: + print( + f" ---- TIME ON AIR ERROR expected {expected_time_on_air}ms " + + f"got {time_on_air_us}us" + ) + failures.add((modem, modem_settings, packet_config)) + + print("************************") + + print(f"\n{len(failures)}/{tests} tests failed") + if failures: + print("FAILURES:") + for f in failures: + print(f) + raise SystemExit(1) + print("SUCCESS") + + +class DummySPI: + # Dummy SPI Interface allows us to use normal constructors + # + # Reading will always return the 'always_read' value + def __init__(self, always_read=0x00): + self.always_read = always_read + + def write_readinto(self, _wrbuf, rdbuf): + for i in range(len(rdbuf)): + rdbuf[i] = self.always_read + + +class DummyPin: + # Dummy Pin interface allows us to use normal constructors + def __init__(self): + pass + + def __call__(self, _=None): + pass + + +# Copies of the functions used to calculate SX1276 SF5, SF6 test case symbol counts. +# (see comments above). +# +# These are written as closely to the SX1276 datasheet "Time on air" section as +# possible, quite different from the BaseModem implementation. + + +def sx1276_n_payload(pl, sf, ih, de, cr, crc): + import math + + ceil_arg = 8 * pl - 4 * sf + 28 + 16 * crc - 20 * ih + ceil_arg /= 4 * (sf - 2 * de) + return 8 + max(math.ceil(ceil_arg) * (cr + 4), 0) + + +def sx1276_n_syms(pl, sf, ih, de, cr, crc, n_preamble): + return sx1276_n_payload(pl, sf, ih, de, cr, crc) + n_preamble + 4.25 + + +if __name__ == "__main__": + main() diff --git a/micropython/mip/manifest.py b/micropython/mip/manifest.py index f6d47e228..9fb94ebcb 100644 --- a/micropython/mip/manifest.py +++ b/micropython/mip/manifest.py @@ -1,5 +1,5 @@ -metadata(version="0.2.0", description="On-device package installer for network-capable boards") +metadata(version="0.4.1", description="On-device package installer for network-capable boards") -require("urequests") +require("requests") package("mip", opt=3) diff --git a/micropython/mip/mip/__init__.py b/micropython/mip/mip/__init__.py index 0593e2e0f..7c0fb4d3a 100644 --- a/micropython/mip/mip/__init__.py +++ b/micropython/mip/mip/__init__.py @@ -1,13 +1,16 @@ # MicroPython package installer # MIT license; Copyright (c) 2022 Jim Mussared -import urequests as requests +from micropython import const +import requests import sys _PACKAGE_INDEX = const("https://micropython.org/pi/v2") _CHUNK_SIZE = 128 +allowed_mip_url_prefixes = ("http://", "https://", "github:", "gitlab:") + # This implements os.makedirs(os.dirname(path)) def _ensure_path_exists(path): @@ -72,6 +75,18 @@ def _rewrite_url(http://github.com/url,%20branch=None): + "/" + "/".join(url[2:]) ) + elif url.startswith("gitlab:"): + url = url[7:].split("/") + url = ( + "https://gitlab.com/" + + url[0] + + "/" + + url[1] + + "/-/raw/" + + branch + + "/" + + "/".join(url[2:]) + ) return url @@ -111,8 +126,12 @@ def _install_json(package_json_url, index, target, version, mpy): if not _download_file(file_url, fs_target_path): print("File not found: {} {}".format(target_path, short_hash)) return False + base_url = package_json_url.rpartition("/")[0] for target_path, url in package_json.get("urls", ()): fs_target_path = target + "/" + target_path + is_full_url = any(url.startswith(p) for p in allowed_mip_url_prefixes) + if base_url and not is_full_url: + url = f"{base_url}/{url}" # Relative URLs if not _download_file(_rewrite_url(http://github.com/url,%20version), fs_target_path): print("File not found: {} {}".format(target_path, url)) return False @@ -123,11 +142,7 @@ def _install_json(package_json_url, index, target, version, mpy): def _install_package(package, index, target, version, mpy): - if ( - package.startswith("http://") - or package.startswith("https://") - or package.startswith("github:") - ): + if any(package.startswith(p) for p in allowed_mip_url_prefixes): if package.endswith(".py") or package.endswith(".mpy"): print("Downloading {} to {}".format(package, target)) return _download_file( @@ -156,7 +171,7 @@ def _install_package(package, index, target, version, mpy): def install(package, index=None, target=None, version=None, mpy=True): if not target: for p in sys.path: - if p.endswith("/lib"): + if not p.startswith("/rom") and p.endswith("/lib"): target = p break else: diff --git a/micropython/net/ntptime/manifest.py b/micropython/net/ntptime/manifest.py index fa444ec75..15f832966 100644 --- a/micropython/net/ntptime/manifest.py +++ b/micropython/net/ntptime/manifest.py @@ -1 +1,3 @@ +metadata(description="NTP client.", version="0.1.1") + module("ntptime.py", opt=3) diff --git a/micropython/net/ntptime/ntptime.py b/micropython/net/ntptime/ntptime.py index ff0d9d202..d77214d1d 100644 --- a/micropython/net/ntptime/ntptime.py +++ b/micropython/net/ntptime/ntptime.py @@ -1,13 +1,6 @@ -import utime - -try: - import usocket as socket -except: - import socket -try: - import ustruct as struct -except: - import struct +from time import gmtime +import socket +import struct # The NTP host can be configured at runtime by doing: ntptime.host = 'myhost.org' host = "pool.ntp.org" @@ -22,13 +15,38 @@ def time(): s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: s.settimeout(timeout) - res = s.sendto(NTP_QUERY, addr) + s.sendto(NTP_QUERY, addr) msg = s.recv(48) finally: s.close() val = struct.unpack("!I", msg[40:44])[0] - EPOCH_YEAR = utime.gmtime(0)[0] + # 2024-01-01 00:00:00 converted to an NTP timestamp + MIN_NTP_TIMESTAMP = 3913056000 + + # Y2036 fix + # + # The NTP timestamp has a 32-bit count of seconds, which will wrap back + # to zero on 7 Feb 2036 at 06:28:16. + # + # We know that this software was written during 2024 (or later). + # So we know that timestamps less than MIN_NTP_TIMESTAMP are impossible. + # So if the timestamp is less than MIN_NTP_TIMESTAMP, that probably means + # that the NTP time wrapped at 2^32 seconds. (Or someone set the wrong + # time on their NTP server, but we can't really do anything about that). + # + # So in that case, we need to add in those extra 2^32 seconds, to get the + # correct timestamp. + # + # This means that this code will work until the year 2160. More precisely, + # this code will not work after 7th Feb 2160 at 06:28:15. + # + if val < MIN_NTP_TIMESTAMP: + val += 0x100000000 + + # Convert timestamp from NTP format to our internal format + + EPOCH_YEAR = gmtime(0)[0] if EPOCH_YEAR == 2000: # (date(2000, 1, 1) - date(1900, 1, 1)).days * 24*60*60 NTP_DELTA = 3155673600 @@ -46,5 +64,5 @@ def settime(): t = time() import machine - tm = utime.gmtime(t) + tm = gmtime(t) machine.RTC().datetime((tm[0], tm[1], tm[2], tm[6] + 1, tm[3], tm[4], tm[5], 0)) diff --git a/micropython/net/webrepl/manifest.py b/micropython/net/webrepl/manifest.py index 6d1a31421..20527db4f 100644 --- a/micropython/net/webrepl/manifest.py +++ b/micropython/net/webrepl/manifest.py @@ -1,2 +1,4 @@ +metadata(description="WebREPL server.", version="0.1.0") + module("webrepl.py", opt=3) module("webrepl_setup.py", opt=3) diff --git a/micropython/net/webrepl/webrepl.py b/micropython/net/webrepl/webrepl.py index 56767d8b7..00da8155c 100644 --- a/micropython/net/webrepl/webrepl.py +++ b/micropython/net/webrepl/webrepl.py @@ -1,6 +1,7 @@ # This module should be imported from REPL, not run from command line. import binascii import hashlib +from micropython import const import network import os import socket @@ -101,7 +102,7 @@ def setup_conn(port, accept_handler): listen_s.listen(1) if accept_handler: listen_s.setsockopt(socket.SOL_SOCKET, 20, accept_handler) - for i in (network.AP_IF, network.STA_IF): + for i in (network.WLAN.IF_AP, network.WLAN.IF_STA): iface = network.WLAN(i) if iface.active(): print("WebREPL server started on http://%s:%d/" % (iface.ifconfig()[0], port)) diff --git a/micropython/senml/README.md b/micropython/senml/README.md new file mode 100644 index 000000000..9b79cbf67 --- /dev/null +++ b/micropython/senml/README.md @@ -0,0 +1,12 @@ +# Introduction + +The SenML library helps you create and parse [senml documents](https://tools.ietf.org/html/draft-ietf-core-senml-13) +in both json and cbor format. + +# key features + +- Object oriented design. +- built in support for [senml's unit registry](https://tools.ietf.org/html/draft-ietf-core-senml-12#section-12.1) +- extensible for new data types +- direct support to read/write in json and cbor format. +- automatically adjusts record data with respect to base time, base value & base sum. diff --git a/micropython/senml/docs/_config.yml b/micropython/senml/docs/_config.yml new file mode 100644 index 000000000..c74188174 --- /dev/null +++ b/micropython/senml/docs/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-slate \ No newline at end of file diff --git a/micropython/senml/docs/index.md b/micropython/senml/docs/index.md new file mode 100644 index 000000000..91ed7fe99 --- /dev/null +++ b/micropython/senml/docs/index.md @@ -0,0 +1,13 @@ +Welcome to the API documet site for the micro-python SenML library. + +The following api sections are available: + +- [senml-base](./senml_base): the base class for all senml objects. +- [senml-pack](./senml_pack): the class that represents root documents. +- [senml-record](./senml_record): the class that stores sensor measurements +- [senml-unit](./senml_unit): the list of all unit names that can be used. + + + +Copyright (c) 2018 KPN +Copyright (c) 2023 MicroPython diff --git a/micropython/senml/docs/senml_base.md b/micropython/senml/docs/senml_base.md new file mode 100644 index 000000000..feeff22e2 --- /dev/null +++ b/micropython/senml/docs/senml_base.md @@ -0,0 +1,8 @@ + +# senml_base Module + + +## senml_base.SenmlBase Objects + + +the base class for all senml objects. diff --git a/micropython/senml/docs/senml_pack.md b/micropython/senml/docs/senml_pack.md new file mode 100644 index 000000000..4a51cff78 --- /dev/null +++ b/micropython/senml/docs/senml_pack.md @@ -0,0 +1,216 @@ + +# senml_pack Module + + +## senml_pack.SenmlPack Objects + + +represents a senml pack object. This can contain multiple records but also other (child) pack objects. +When the pack object only contains records, it represents the data of a device. +If the pack object has child pack objects, then it represents a gateway + +### __enter__ + +```Python +__enter__(self) +``` + +for supporting the 'with' statement + + +_returns_: self + +### __exit__ + +```Python +__exit__(self, exc_type, exc_val, exc_tb) +``` + +when destroyed in a 'with' statement, make certain that the item is removed from the parent list. + + +_returns_: None + +### __init__ + +```Python +__init__(self, name, callback=None) +``` + +initialize the object + +_parameters:_ + +- `name:` {string} the name of the pack + +### __iter__ + +```Python +__iter__(self) +``` + + + +### add + +```Python +adds the item to the list of records +``` + + +_parameters:_ + +- `item:` {SenmlRecord} the item that needs to be added to the pack + + +_returns_: None + +### base_sum + +the base sum of the pack. + + +_returns_: a number + +### base_time + +Get the base time assigned to this pack object. +While rendering, this value will be subtracted from the value of the records. + + +_returns_: unix time stamp representing the base time + +### base_value + +the base value of the pack. The value of the records will be subtracted by this value during rendering. +While parsing, this value is added to the value of the records. + + +_returns_: a number + +### clear + +```Python +clear(self) +``` +clear the list of the pack + + + +_returns_: None + +### do_actuate + +```Python +do_actuate(self, raw, naming_map, device=None) +``` + +called while parsing incoming data for a record that is not yet part of this pack object. +adds a new record and raises the actuate callback of the pack with the newly created record as argument + +_parameters:_ + +- naming_map: +- `device:` optional: if the device was not found +- `raw:` the raw record definition, as found in the json structure. this still has invalid labels. + + +_returns_: None + +### from_cbor + +```Python +from_cbor(self, data) +``` + +parse a cbor data byte array to a senml pack structure. + +_parameters:_ + +- `data:` a byte array. + + +_returns_: None + +### from_json + +```Python +from_json(self, data) +``` + +parse a json string and convert it to a senml pack structure + +_parameters:_ + +- `data:` a string containing json data. + + +_returns_: None, will call the appropriate callback functions. + + + +### remove + +```Python +remove(self, item) +``` +removes the item from the pack + + +_parameters:_ + +- `item:` {SenmlRecord} the item that needs to be removed + + +_returns_: None + +### to_cbor + +```Python +to_cbor(self) +``` + +render the content of this object to a cbor byte array + + +_returns_: a byte array + +### to_json + +```Python +to_json(self) +``` + +render the content of this object to a string. + + +_returns_: a string representing the senml pack object + +## senml_pack.SenmlPackIterator Objects + + +an iterator to walk over all records in a pack + +### __init__ + +```Python +__init__(self, list) +``` + + + +### __iter__ + +```Python +__iter__(self) +``` + + + +### __next__ + +```Python +__next__(self) +``` + + diff --git a/micropython/senml/docs/senml_record.md b/micropython/senml/docs/senml_record.md new file mode 100644 index 000000000..6bac549a5 --- /dev/null +++ b/micropython/senml/docs/senml_record.md @@ -0,0 +1,86 @@ + +# senml_record Module + + +## senml_record.SenmlRecord Objects + + +represents a single value in a senml pack object + +### __enter__ + +```Python +__enter__(self) +``` + +for supporting the 'with' statement + + +_returns_: self + +### __exit__ + +```Python +__exit__(self, exc_type, exc_val, exc_tb) +``` + +when destroyed in a 'with' statement, make certain that the item is removed from the parent list. + + +_returns_: None + +### __init__ + +```Python +__init__(self, name, **kwargs) +``` + +create a new senml record + +_parameters:_ + +- `kwargs:` optional parameters: + - value: the value to store in the record + - time: the timestamp to use (when was the value measured) + - name: the name of hte record + - unit: unit value + - sum: sum value + - update_time: max time before sensor will provide an updated reading + - callback: a callback function taht will be called when actuator data has been found. Expects no params + +### do_actuate + +```Python +do_actuate(self, raw, naming_map) +``` + +called when a raw senml record was found for this object. Stores the data and if there is a callback, calls it. + +_parameters:_ + +- `raw:` raw senml object + + +_returns_: None + +### sum + + + +### time + +get the time at which the measurement for the record was taken. + + +_returns_: a unix time stamp. This is the absolute value, not adjusted to the base time of the pack. + +### update_time + +get the time at which the next measurement is expected to be taken for this record. + + +_returns_: a unix time stamp. This is the absolute value, not adjusted to the base time of the pack. + +### value + +get the value currently assigned to the object diff --git a/micropython/senml/docs/senml_unit.md b/micropython/senml/docs/senml_unit.md new file mode 100644 index 000000000..816c40679 --- /dev/null +++ b/micropython/senml/docs/senml_unit.md @@ -0,0 +1,183 @@ + +# senml_unit Module + + +## Functions + + + +## senml_unit.SenmlUnits Objects + + + + +##### `SENML_UNIT_ACCELERATION` + + +##### `SENML_UNIT_AMPERE` + + +##### `SENML_UNIT_BEATS` + + +##### `SENML_UNIT_BECQUEREL` + + +##### `SENML_UNIT_BEL` + + +##### `SENML_UNIT_BIT` + + +##### `SENML_UNIT_BIT_PER_SECOND` + + +##### `SENML_UNIT_BPM` + + +##### `SENML_UNIT_CANDELA` + + +##### `SENML_UNIT_CANDELA_PER_SQUARE_METER` + + +##### `SENML_UNIT_COULOMB` + + +##### `SENML_UNIT_COUNTER` + + +##### `SENML_UNIT_CUBIC_METER` + + +##### `SENML_UNIT_CUBIC_METER_PER_SECOND` + + +##### `SENML_UNIT_DECIBEL` + + +##### `SENML_UNIT_DECIBEL_RELATIVE_TO_1_W` + + +##### `SENML_UNIT_DEGREES_CELSIUS` + + +##### `SENML_UNIT_DEGREES_LATITUDE` + + +##### `SENML_UNIT_DEGREES_LONGITUDE` + + +##### `SENML_UNIT_EVENT_RATE_PER_MINUTE` + + +##### `SENML_UNIT_EVENT_RATE_PER_SECOND` + + +##### `SENML_UNIT_FARAD` + + +##### `SENML_UNIT_GRAM` + + +##### `SENML_UNIT_GRAY` + + +##### `SENML_UNIT_HENRY` + + +##### `SENML_UNIT_HERTZ` + + +##### `SENML_UNIT_JOULE` + + +##### `SENML_UNIT_KATAL` + + +##### `SENML_UNIT_KELVIN` + + +##### `SENML_UNIT_KILOGRAM` + + +##### `SENML_UNIT_LITER` + + +##### `SENML_UNIT_LITER_PER_SECOND` + + +##### `SENML_UNIT_LUMEN` + + +##### `SENML_UNIT_LUX` + + +##### `SENML_UNIT_METER` + + +##### `SENML_UNIT_MOLE` + + +##### `SENML_UNIT_NEWTON` + + +##### `SENML_UNIT_OHM` + + +##### `SENML_UNIT_PASCAL` + + +##### `SENML_UNIT_PERCENTAGE_REMAINING_BATTERY_LEVEL` + + +##### `SENML_UNIT_PH` + + +##### `SENML_UNIT_RADIAN` + + +##### `SENML_UNIT_RATIO` + + +##### `SENML_UNIT_RELATIVE_HUMIDITY` + + +##### `SENML_UNIT_SECOND` + + +##### `SENML_UNIT_SECONDS_REMAINING_BATTERY_LEVEL` + + +##### `SENML_UNIT_SIEMENS` + + +##### `SENML_UNIT_SIEMENS_PER_METER` + + +##### `SENML_UNIT_SIEVERT` + + +##### `SENML_UNIT_SQUARE_METER` + + +##### `SENML_UNIT_STERADIAN` + + +##### `SENML_UNIT_TESLA` + + +##### `SENML_UNIT_VELOCITY` + + +##### `SENML_UNIT_VOLT` + + +##### `SENML_UNIT_WATT` + + +##### `SENML_UNIT_WATT_PER_SQUARE_METER` + + +##### `SENML_UNIT_WEBER` + diff --git a/micropython/senml/examples/actuator.py b/micropython/senml/examples/actuator.py new file mode 100644 index 000000000..2fac474cd --- /dev/null +++ b/micropython/senml/examples/actuator.py @@ -0,0 +1,66 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from senml import * + + +def do_actuate(record): + """ + called when actuate_me receives a value. + :return: None + """ + print(record.value) + + +def generic_callback(record, **kwargs): + """ + a generic callback, attached to the device. Called when a record is found that has not yet been registered + in the pack. When this callback is called, the record will already be added to the pack. + :param record: the newly found record. + :return: None + """ + print("found record: " + record.name) + print("with value: " + str(record.value)) + + +pack = SenmlPack("device_name", generic_callback) +actuate_me = SenmlRecord("actuator", callback=do_actuate) + +pack.add(actuate_me) + +json_data = '[{"bn": "device_name", "n":"actuator", "v": 10 }]' +print(json_data) +pack.from_json(json_data) + +json_data = ( + '[{"bn": "device_name", "n":"actuator", "v": 20 }, {"n": "another_actuator", "vs": "a value"}]' +) +print(json_data) +pack.from_json(json_data) + +print('[{"bn": "device_name", "n":"temp", "v": 20, "u": "Cel" }]') +# this represents the cbor json struct: [{-2: "device_name", 0: "temp", 1: "Cel", 2: 20}] +cbor_data = bytes.fromhex("81A4216B6465766963655F6E616D65006474656D70016343656C0214") +pack.from_cbor(cbor_data) diff --git a/micropython/senml/examples/base.py b/micropython/senml/examples/base.py new file mode 100644 index 000000000..6a49cfdd2 --- /dev/null +++ b/micropython/senml/examples/base.py @@ -0,0 +1,46 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from senml import * +import time + + +pack = SenmlPack("device_name") +temp = SenmlRecord("temperature", unit=SenmlUnits.SENML_UNIT_DEGREES_CELSIUS, value=23.5) +door_pos = SenmlRecord("doorPos", update_time=20, value=True) +int_val = SenmlRecord("int_val", sum=100) + +pack.add(temp) +pack.add(door_pos) +pack.add(int_val) + +pack.base_time = time.time() +pack.base_value = 5 +pack.base_sum = 50 +time.sleep(2) +temp.time = time.time() + + +print(pack.to_json()) diff --git a/micropython/senml/examples/basic.py b/micropython/senml/examples/basic.py new file mode 100644 index 000000000..3f3ed6150 --- /dev/null +++ b/micropython/senml/examples/basic.py @@ -0,0 +1,38 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from senml import * +import time + + +pack = SenmlPack("device") + +while True: + with SenmlRecord( + "test", value=1 + ) as rec: # use a with statement to automatically remove the item from the list when it goes out of scope + pack.add(rec) + print(pack.to_json()) + time.sleep(1) diff --git a/micropython/senml/examples/basic2.py b/micropython/senml/examples/basic2.py new file mode 100644 index 000000000..ca53b4a6e --- /dev/null +++ b/micropython/senml/examples/basic2.py @@ -0,0 +1,44 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from senml import * +import time + + +pack = SenmlPack("device_name") +temp = SenmlRecord("temperature", unit=SenmlUnits.SENML_UNIT_DEGREES_CELSIUS, value=23.5) +door_pos = SenmlRecord("doorPos", update_time=20, value=True) +str_val = SenmlRecord("str val") + +pack.add(temp) +pack.add(door_pos) +pack.add(str_val) + +while True: + temp.value = temp.value + 1.1 + door_pos.value = not door_pos.value + str_val.value = "test" + print(pack.to_json()) + time.sleep(1) diff --git a/micropython/senml/examples/basic_cbor.py b/micropython/senml/examples/basic_cbor.py new file mode 100644 index 000000000..b9d9d620b --- /dev/null +++ b/micropython/senml/examples/basic_cbor.py @@ -0,0 +1,41 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from senml import * +import time +import cbor2 + +pack = SenmlPack("device_name") + +while True: + with SenmlRecord( + "test", value=10 + ) as rec: # use a with statement to automatically remove the item from the list when it goes out of scope, generate a value for the record + pack.add(rec) + cbor_val = pack.to_cbor() + print(cbor_val) + print(cbor_val.hex()) + print(cbor2.loads(cbor_val)) # convert to string again so we can print it. + time.sleep(1) diff --git a/micropython/senml/examples/custom_record.py b/micropython/senml/examples/custom_record.py new file mode 100644 index 000000000..1e83ea06b --- /dev/null +++ b/micropython/senml/examples/custom_record.py @@ -0,0 +1,132 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from senml import * + +import time + + +class Coordinates(SenmlRecord): + def __init__(self, name, **kwargs): + """overriding the init function so we can initiate the 3 senml records that will represent lat,lon, alt""" + self._lat = SenmlRecord( + "lattitude", unit=SenmlUnits.SENML_UNIT_DEGREES_LATITUDE + ) # create these befor calling base constructor so that all can be init correctly from constructor + self._lon = SenmlRecord("longitude", unit=SenmlUnits.SENML_UNIT_DEGREES_LONGITUDE) + self._alt = SenmlRecord("altitude", unit=SenmlUnits.SENML_UNIT_METER) + super(Coordinates, self).__init__( + name, **kwargs + ) # need to call base init, to make certain all is ok. + + def _check_value_type(self, value): + """overriding the check on value type to make certain that only an array with 3 values is assigned: lat,lon/alt""" + if value is not None: + if not isinstance(value, list): + raise Exception("invalid data type: array with 3 elements expected lat, lon, alt") + + def _build_rec_dict(self, naming_map, appendTo): + """ + override the rendering of the senml data objects. These will be converted to json or cbor + :param naming_map: {dictionary} a map that determines the field names, these are different for json vs cbor + :param appendTo: {list} the result list + :return: None + """ + self._lat._build_rec_dict(naming_map, appendTo) + self._lon._build_rec_dict(naming_map, appendTo) + self._alt._build_rec_dict(naming_map, appendTo) + + @SenmlRecord.value.setter + def value(self, value): + """set the current value. + this is overridden so we can pass on the values to the internal objects. It's also stored in the parent + so that a 'get-value' still returns the array. + """ + self._value = ( + value # micropython doesn't support calling setter of parent property, do it manually + ) + if value: + self._lat.value = value[0] + self._lon.value = value[1] + self._alt.value = value[2] + else: + self._lat.value = None + self._lon.value = None + self._alt.value = None + + @SenmlRecord.time.setter + def time(self, value): + """set the time stamp. + this is overridden so we can pass on the values to the internal objects. + """ + self._check_number_type( + value, "time" + ) # micropython doesn't support calling setter of parent property, do it manually + self._time = value + self._lat.time = value + self._lon.time = value + self._alt.time = value + + @SenmlRecord.update_time.setter + def update_time(self, value): + """set the time stamp. + this is overridden so we can pass on the values to the internal objects. + """ + self._check_number_type( + value, "update_time" + ) # micropython doesn't support calling setter of parent property, do it manually + self._update_time = value + self._lat.update_time = value + self._lon.update_time = value + self._alt.update_time = value + + @SenmlRecord._parent.setter + def _parent(self, value): + """set the time stamp. + this is overridden so we can pass on the values to the internal objects. + This is needed so that the child objects can correctly take base time (optionally also base-sum, base-value) into account + """ + self.__parent = ( + value # micropython doesn't support calling setter of parent property, do it manually + ) + self._lat._parent = value + self._lon._parent = value + self._alt._parent = value + + +pack = SenmlPack("device_name") +loc = Coordinates("location") +loc2 = Coordinates("location", value=[52.0259, 5.4775, 230]) +pack.add(loc) +pack.add(loc2) + +print(loc._parent.name) + +loc.value = [51.0259, 4.4775, 10] +print(pack.to_json()) + +pack.base_time = time.time() # set a base time +time.sleep(2) +loc.time = time.time() # all child objects will receive the time value +print(pack.to_json()) diff --git a/micropython/senml/examples/gateway.py b/micropython/senml/examples/gateway.py new file mode 100644 index 000000000..e1827ff2d --- /dev/null +++ b/micropython/senml/examples/gateway.py @@ -0,0 +1,49 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from senml import * +import time + +gateway_pack = SenmlPack("gateway") + +dev1_pack = SenmlPack("dev1") +dev2_pack = SenmlPack("dev2") + +temp = SenmlRecord("temperature", unit=SenmlUnits.SENML_UNIT_DEGREES_CELSIUS, value=23.5) +door_pos = SenmlRecord("doorPos", update_time=20, value=True) +str_val = SenmlRecord("str val") + +gateway_pack.add(temp) +gateway_pack.add(dev1_pack) +gateway_pack.add(dev2_pack) +dev1_pack.add(door_pos) +dev2_pack.add(str_val) + +while True: + temp.value = temp.value + 1.1 + door_pos.value = not door_pos.value + str_val.value = "test" + print(gateway_pack.to_json()) + time.sleep(1) diff --git a/micropython/senml/examples/gateway_actuators.py b/micropython/senml/examples/gateway_actuators.py new file mode 100644 index 000000000..a7e5b378c --- /dev/null +++ b/micropython/senml/examples/gateway_actuators.py @@ -0,0 +1,74 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from senml import * + + +def do_actuate(record): + """ + called when actuate_me receives a value. + :return: None + """ + print("for known device: ") + print(record.value) + + +def device_callback(record, **kwargs): + """ + a generic callback, attached to the device. Called when a record is found that has not yet been registered + in the pack. When this callback is called, the record will already be added to the pack. + :param kwargs: optional extra parameters + :param record: the newly found record. + :return: None + """ + print("found record: " + record.name) + print("with value: " + record.value) + + +def gateway_callback(record, **kwargs): + """ + a generic callback, attached to the device. Called when a record is found that has not yet been registered + in the pack. When this callback is called, the record will already be added to the pack. + :param record: the newly found record. + :param kwargs: optional extra parameters (device can be found here) + :return: None + """ + if "device" in kwargs and kwargs["device"] is not None: + print("for device: " + kwargs["device"].name) + else: + print("for gateway: ") + print("found record: " + record.name) + print("with value: " + str(record.value)) + + +gateway = SenmlPack("gateway_name", gateway_callback) +device = SenmlPack("device_name", device_callback) +actuate_me = SenmlRecord("actuator", callback=do_actuate) + +gateway.add(device) +device.add(actuate_me) +gateway.from_json( + '[{"bn": "gateway_name", "n":"temp", "v": 22},{"n": "gateway_actuator", "vb": true}, {"bn": "device_name", "n":"actuator", "v": 20 }, {"n": "another_actuator", "vs": "a value"}, {"bn": "device_2", "n":"temp", "v": 20 }, {"n": "actuator2", "vs": "value2"}]' +) diff --git a/micropython/senml/examples/supported_data_types.py b/micropython/senml/examples/supported_data_types.py new file mode 100644 index 000000000..94976bb66 --- /dev/null +++ b/micropython/senml/examples/supported_data_types.py @@ -0,0 +1,52 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from senml import * +import time + +pack = SenmlPack("device_name") + +double_val = SenmlRecord("double", value=23.5) +int_val = SenmlRecord("int", value=23) +bool_val = SenmlRecord("bool", value=True) +str_val = SenmlRecord("str val", value="test") +bytes_val = SenmlRecord("bytes", value=bytearray(b"00 1e 05 ff")) + +# invalid value +try: + invalid = SenmlRecord("invalid", value={"a": 1}) +except Exception as error: + print(error) + + +pack.add(double_val) +pack.add(int_val) +pack.add(bool_val) +pack.add(str_val) +pack.add(bytes_val) + +while True: + print(pack.to_json()) + time.sleep(1) diff --git a/micropython/senml/manifest.py b/micropython/senml/manifest.py new file mode 100644 index 000000000..f4743075a --- /dev/null +++ b/micropython/senml/manifest.py @@ -0,0 +1,9 @@ +metadata( + description="SenML serialisation for MicroPython.", + version="0.1.1", + pypi_publish="micropython-senml", +) + +require("cbor2") + +package("senml") diff --git a/micropython/senml/senml/__init__.py b/micropython/senml/senml/__init__.py new file mode 100644 index 000000000..908375fdb --- /dev/null +++ b/micropython/senml/senml/__init__.py @@ -0,0 +1,29 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from .senml_base import SenmlBase +from .senml_pack import SenmlPack +from .senml_record import SenmlRecord +from .senml_unit import SenmlUnits diff --git a/micropython/senml/senml/senml_base.py b/micropython/senml/senml/senml_base.py new file mode 100644 index 000000000..b277c9477 --- /dev/null +++ b/micropython/senml/senml/senml_base.py @@ -0,0 +1,30 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + + +class SenmlBase(object): + """ + the base class for all senml objects. + """ diff --git a/micropython/senml/senml/senml_pack.py b/micropython/senml/senml/senml_pack.py new file mode 100644 index 000000000..5a0554467 --- /dev/null +++ b/micropython/senml/senml/senml_pack.py @@ -0,0 +1,358 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from senml.senml_record import SenmlRecord +from senml.senml_base import SenmlBase +import json +import cbor2 + + +class SenmlPackIterator: + """an iterator to walk over all records in a pack""" + + def __init__(self, list): + self._list = list + self._index = 0 + + def __iter__(self): + return self + + def __next__(self): + if self._index < len(self._list): + res = self._list[self._index] + self._index += 1 + return res + else: + raise StopIteration + + +class SenmlPack(SenmlBase): + """ + represents a sneml pack object. This can contain multiple records but also other (child) pack objects. + When the pack object only contains records, it represents the data of a device. + If the pack object has child pack objects, then it represents a gateway + """ + + json_mappings = { + "bn": "bn", + "bt": "bt", + "bu": "bu", + "bv": "bv", + "bs": "bs", + "n": "n", + "u": "u", + "v": "v", + "vs": "vs", + "vb": "vb", + "vd": "vd", + "s": "s", + "t": "t", + "ut": "ut", + } + + def __init__(self, name, callback=None): + """ + initialize the object + :param name: {string} the name of the pack + """ + self._data = [] + self.name = name + self._base_value = None + self._base_time = None + self._base_sum = None + self.base_unit = None + self._parent = None # a pack can also be the child of another pack. + self.actuate = callback # actuate callback function + + def __iter__(self): + return SenmlPackIterator(self._data) + + def __enter__(self): + """ + for supporting the 'with' statement + :return: self + """ + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + when destroyed in a 'with' statement, make certain that the item is removed from the parent list. + :return: None + """ + if self._parent: + self._parent.remove(self) + + @property + def base_value(self): + """ + the base value of the pack. + :return: a number + """ + return self._base_value + + @base_value.setter + def base_value(self, value): + """ + set the base value. + :param value: only number allowed + :return: + """ + self._check_value_type(value, "base_value") + self._base_value = value + + @property + def base_sum(self): + """ + the base sum of the pack. + :return: a number + """ + return self._base_sum + + @base_sum.setter + def base_sum(self, value): + """ + set the base value. + :param value: only number allowed + :return: + """ + self._check_value_type(value, "base_sum") + self._base_sum = value + + @property + def base_time(self): + return self._base_time + + @base_time.setter + def base_time(self, value): + self._check_value_type(value, "base_time") + self._base_time = value + + def _check_value_type(self, value, field_name): + """ + checks if the type of value is allowed for senml + :return: None, raisee exception if not ok. + """ + if value is not None: + if not (isinstance(value, int) or isinstance(value, float)): + raise Exception("invalid type for " + field_name + ", only numbers allowed") + + def from_json(self, data): + """ + parse a json string and convert it to a senml pack structure + :param data: a string containing json data. + :return: None, will r + """ + records = json.loads(data) # load the raw senml data + self._process_incomming_data(records, SenmlPack.json_mappings) + + def _process_incomming_data(self, records, naming_map): + """ + generic processor for incomming data (actuators. + :param records: the list of raw senml data, parsed from a json or cbor structure + :param naming_map: translates cbor to json field names (when needed). + :return: None + """ + cur_pack_el = self + new_pack = False + for item in records: + if naming_map["bn"] in item: # ref to a pack element, either this or a child pack. + if item[naming_map["bn"]] != self.name: + pack_el = [x for x in self._data if x.name == item[naming_map["bn"]]] + else: + pack_el = [self] + if len(pack_el) > 0: + cur_pack_el = pack_el[0] + new_pack = False + else: + device = SenmlPack(item[naming_map["bn"]]) + self._data.append(device) + cur_pack_el = device + new_pack = True + + if ( + naming_map["bv"] in item + ): # need to copy the base value assigned to the pack element so we can do proper conversion for actuators. + cur_pack_el.base_value = item[naming_map["bv"]] + + rec_el = [x for x in cur_pack_el._data if x.name == item[naming_map["n"]]] + if len(rec_el) > 0: + rec_el[0].do_actuate(item, naming_map) + elif new_pack: + self.do_actuate(item, naming_map, cur_pack_el) + else: + cur_pack_el.do_actuate(item, naming_map) + else: + rec_el = [x for x in self._data if x.name == item[naming_map["n"]]] + if len(rec_el) > 0: + rec_el[0].do_actuate(item, naming_map) + elif new_pack: + self.do_actuate(item, naming_map, cur_pack_el) + else: + cur_pack_el.do_actuate(item, naming_map) + + def do_actuate(self, raw, naming_map, device=None): + """ + called while parsing incoming data for a record that is not yet part of this pack object. + adds a new record and raises the actuate callback of the pack with the newly created record as argument + :param naming_map: + :param device: optional: if the device was not found + :param raw: the raw record definition, as found in the json structure. this still has invalid labels. + :return: None + """ + rec = SenmlRecord(raw[naming_map["n"]]) + if device: + device.add(rec) + rec._from_raw(raw, naming_map) + if self.actuate: + self.actuate(rec, device=device) + else: + self.add(rec) + rec._from_raw(raw, naming_map) + if self.actuate: + self.actuate(rec, device=None) + + def to_json(self): + """ + render the content of this object to a string. + :return: a string representing the senml pack object + """ + converted = [] + self._build_rec_dict(SenmlPack.json_mappings, converted) + return json.dumps(converted) + + def _build_rec_dict(self, naming_map, appendTo): + """ + converts the object to a senml object with the proper naming in place. + This can be recursive: a pack can contain other packs. + :param naming_map: a dictionary used to pick the correct field names for either senml json or senml cbor + :return: + """ + internalList = [] + for item in self._data: + item._build_rec_dict(naming_map, internalList) + if len(internalList) > 0: + first_rec = internalList[0] + else: + first_rec = {} + internalList.append(first_rec) + + if self.name: + first_rec[naming_map["bn"]] = self.name + if self.base_value: + first_rec[naming_map["bv"]] = self.base_value + if self.base_unit: + first_rec[naming_map["bu"]] = self.base_unit + if self.base_sum: + first_rec[naming_map["bs"]] = self.base_sum + if self.base_time: + first_rec[naming_map["bt"]] = self.base_time + appendTo.extend(internalList) + + def from_cbor(self, data): + """ + parse a cbor data byte array to a senml pack structure. + :param data: a byte array. + :return: None + """ + records = cbor2.loads(data) # load the raw senml data + naming_map = { + "bn": -2, + "bt": -3, + "bu": -4, + "bv": -5, + "bs": -16, + "n": 0, + "u": 1, + "v": 2, + "vs": 3, + "vb": 4, + "vd": 8, + "s": 5, + "t": 6, + "ut": 7, + } + self._process_incomming_data(records, naming_map) + + def to_cbor(self): + """ + render the content of this object to a cbor byte array + :return: a byte array + """ + naming_map = { + "bn": -2, + "bt": -3, + "bu": -4, + "bv": -5, + "bs": -16, + "n": 0, + "u": 1, + "v": 2, + "vs": 3, + "vb": 4, + "vd": 8, + "s": 5, + "t": 6, + "ut": 7, + } + converted = [] + self._build_rec_dict(naming_map, converted) + return cbor2.dumps(converted) + + def add(self, item): + """ + adds the item to the list of records + :param item: {SenmlRecord} the item that needs to be added to the pack + :return: None + """ + if not (isinstance(item, SenmlBase)): + raise Exception("invalid type of param, SenmlRecord or SenmlPack expected") + if item._parent is not None: + raise Exception("item is already part of a pack") + + self._data.append(item) + item._parent = self + + def remove(self, item): + """ + removes the item from the list of records + :param item: {SenmlRecord} the item that needs to be removed + :return: None + """ + if not (isinstance(item, SenmlBase)): + raise Exception("invalid type of param, SenmlRecord or SenmlPack expected") + if not item._parent == self: + raise Exception("item is not part of this pack") + + self._data.remove(item) + item._parent = None + + def clear(self): + """ + clear the list of the pack + :return: None + """ + for item in self._data: + item._parent = None + self._data = [] diff --git a/micropython/senml/senml/senml_record.py b/micropython/senml/senml/senml_record.py new file mode 100644 index 000000000..ae40f0f70 --- /dev/null +++ b/micropython/senml/senml/senml_record.py @@ -0,0 +1,246 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import binascii +from senml.senml_base import SenmlBase + + +class SenmlRecord(SenmlBase): + """represents a single value in a senml pack object""" + + def __init__(self, name, **kwargs): + """ + create a new senml record + :param kwargs: optional parameters: + - value: the value to store in the record + - time: the timestamp to use (when was the value measured) + - name: the name of hte record + - unit: unit value + - sum: sum value + - update_time: max time before sensor will provide an updated reading + - callback: a callback function taht will be called when actuator data has been found. Expects no params + """ + self.__parent = None # using double __ cause it's a field for an internal property + self._unit = None # declare and init internal fields + self._value = None + self._time = None + self._sum = None + self._update_time = None + + self._parent = None # internal reference to the parent object + self.name = name + self.unit = kwargs.get("unit", None) + self.value = kwargs.get("value", None) + self.time = kwargs.get("time", None) + self.sum = kwargs.get("sum", None) + self.update_time = kwargs.get("update_time", None) + self.actuate = kwargs.get("callback", None) # actuate callback function + + def __enter__(self): + """ + for supporting the 'with' statement + :return: self + """ + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + when destroyed in a 'with' statement, make certain that the item is removed from the parent list. + :return: None + """ + if self._parent: + self._parent.remove(self) + + def _check_value_type(self, value): + """ + checks if the type of value is allowed for senml + :return: None, raisee exception if not ok. + """ + if value is not None: + if not ( + isinstance(value, bool) + or isinstance(value, int) + or isinstance(value, float) + or isinstance(value, bytearray) + or isinstance(value, str) + ): + raise Exception( + "invalid type for value, only numbers, strings, boolean and byte arrays allowed" + ) + + def _check_number_type(self, value, field_name): + """ + checks if the type of value is allowed for senml + :return: None, raisee exception if not ok. + """ + if value is not None: + if not (isinstance(value, int) or isinstance(value, float)): + raise Exception("invalid type for " + field_name + ", only numbers allowed") + + @property + def value(self): + """get the value currently assigned to the object""" + return self._value + + @value.setter + def value(self, value): + """set the current value. Will not automatically update the time stamp. This has to be done seperatly for more + finegrained control + Note: when the value is a float, you can control rounding in the rendered output by using the function + round() while assigning the value. ex: record.value = round(12.2 / 1.5423, 2) + """ + self._check_value_type(value) + self._value = value + + @property + def time(self): + return self._time + + @time.setter + def time(self, value): + self._check_number_type(value, "time") + self._time = value + + @property + def update_time(self): + return self._update_time + + @update_time.setter + def update_time(self, value): + self._check_number_type(value, "update_time") + self._update_time = value + + @property + def sum(self): + return self._sum + + @sum.setter + def sum(self, value): + self._check_number_type(value, "sum") + self._sum = value + + @property + def _parent(self): + """ + the parent pack object for this record. This is a property so that inheriters can override and do custom + actions when the parent is set (like passing it on to their children + :return: + """ + return self.__parent + + @_parent.setter + def _parent(self, value): + """ + the parent pack object for this record. This is a property so that inheriters can override and do custom + actions when the parent is set (like passing it on to their children + :return: + """ + self.__parent = value + + def _build_rec_dict(self, naming_map, appendTo): + """ + converts the object to a dictionary that can be rendered to senml. + :param naming_map: a dictionary that maps the field names to senml json or senml cbor. keys are in the + form 'n', 'v',... values for 'n' are either 'n' or 0 (number is for cbor) + :return: a senml dictionary representation of the record + """ + result = {} + + if self.name: + result[naming_map["n"]] = self.name + + if self._sum: + if self._parent and self._parent.base_sum: + result[naming_map["s"]] = self._sum - self._parent.base_sum + else: + result[naming_map["s"]] = self._sum + elif isinstance(self._value, bool): + result[naming_map["vb"]] = self._value + elif isinstance(self._value, int) or isinstance(self._value, float): + if self._parent and self._parent.base_value: + result[naming_map["v"]] = self._value - self._parent.base_value + else: + result[naming_map["v"]] = self._value + elif isinstance(self._value, str): + result[naming_map["vs"]] = self._value + elif isinstance(self._value, bytearray): + if ( + naming_map["vd"] == "vd" + ): # neeed to make a distinction between json (needs base64) and cbor (needs binary) + result[naming_map["vd"]] = binascii.b2a_base64(self._value, newline=False).decode( + "utf8" + ) + else: + result[naming_map["vd"]] = self._value + else: + raise Exception("sum or value of type bootl, number, string or byte-array is required") + + if self._time: + if self._parent and self._parent.base_time: + result[naming_map["t"]] = self._time - self._parent.base_time + else: + result[naming_map["t"]] = self._time + + if self.unit: + result[naming_map["u"]] = self.unit + + if self._update_time: + if self._parent and self._parent.base_time: + result[naming_map["ut"]] = self._update_time - self._parent.base_time + else: + result[naming_map["ut"]] = self._update_time + + appendTo.append(result) + + def _from_raw(self, raw, naming_map): + """ + extracts te data from the raw record. Used during parsing of incoming data. + :param raw: a raw senml record which still contains the original field names + :param naming_map: used to map cbor names to json field names + :return: + """ + if naming_map["v"] in raw: + val = raw[naming_map["v"]] + if self._parent and self._parent.base_value: + val += self._parent.base_value + elif naming_map["vs"] in raw: + val = raw[naming_map["vs"]] + elif naming_map["vb"] in raw: + val = raw[naming_map["vb"]] + elif naming_map["vd"] in raw: + val = binascii.a2b_base64(raw[naming_map["vb"]]) + else: + val = None + self.value = val + + def do_actuate(self, raw, naming_map): + """ + called when a raw senml record was found for this object. Stores the data and if there is a callback, calls it. + :param raw: raw senml object + :return: None + """ + self._from_raw(raw, naming_map) + if self.actuate: + self.actuate(self) diff --git a/micropython/senml/senml/senml_unit.py b/micropython/senml/senml/senml_unit.py new file mode 100644 index 000000000..bf7753c4d --- /dev/null +++ b/micropython/senml/senml/senml_unit.py @@ -0,0 +1,89 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + + +def enum(**enums): + return type("Enum", (), enums) + + +SenmlUnits = enum( + SENML_UNIT_METER="m", + SENML_UNIT_KILOGRAM="kg", + SENML_UNIT_GRAM="g", + SENML_UNIT_SECOND="s", + SENML_UNIT_AMPERE="A", + SENML_UNIT_KELVIN="K", + SENML_UNIT_CANDELA="cd", + SENML_UNIT_MOLE="mol", + SENML_UNIT_HERTZ="Hz", + SENML_UNIT_RADIAN="rad", + SENML_UNIT_STERADIAN="sr", + SENML_UNIT_NEWTON="N", + SENML_UNIT_PASCAL="Pa", + SENML_UNIT_JOULE="J", + SENML_UNIT_WATT="W", + SENML_UNIT_COULOMB="C", + SENML_UNIT_VOLT="V", + SENML_UNIT_FARAD="F", + SENML_UNIT_OHM="Ohm", + SENML_UNIT_SIEMENS="S", + SENML_UNIT_WEBER="Wb", + SENML_UNIT_TESLA="T", + SENML_UNIT_HENRY="H", + SENML_UNIT_DEGREES_CELSIUS="Cel", + SENML_UNIT_LUMEN="lm", + SENML_UNIT_LUX="lx", + SENML_UNIT_BECQUEREL="Bq", + SENML_UNIT_GRAY="Gy", + SENML_UNIT_SIEVERT="Sv", + SENML_UNIT_KATAL="kat", + SENML_UNIT_SQUARE_METER="m2", + SENML_UNIT_CUBIC_METER="m3", + SENML_UNIT_LITER="l", + SENML_UNIT_VELOCITY="m/s", + SENML_UNIT_ACCELERATION="m/s2", + SENML_UNIT_CUBIC_METER_PER_SECOND="m3/s", + SENML_UNIT_LITER_PER_SECOND="l/s", + SENML_UNIT_WATT_PER_SQUARE_METER="W/m2", + SENML_UNIT_CANDELA_PER_SQUARE_METER="cd/m2", + SENML_UNIT_BIT="bit", + SENML_UNIT_BIT_PER_SECOND="bit/s", + SENML_UNIT_DEGREES_LATITUDE="lat", + SENML_UNIT_DEGREES_LONGITUDE="lon", + SENML_UNIT_PH="pH", + SENML_UNIT_DECIBEL="db", + SENML_UNIT_DECIBEL_RELATIVE_TO_1_W="dBW", + SENML_UNIT_BEL="Bspl", + SENML_UNIT_COUNTER="count", + SENML_UNIT_RATIO="//", + SENML_UNIT_RELATIVE_HUMIDITY="%RH", + SENML_UNIT_PERCENTAGE_REMAINING_BATTERY_LEVEL="%EL", + SENML_UNIT_SECONDS_REMAINING_BATTERY_LEVEL="EL", + SENML_UNIT_EVENT_RATE_PER_SECOND="1/s", + SENML_UNIT_EVENT_RATE_PER_MINUTE="1/min", + SENML_UNIT_BPM="beat/min", + SENML_UNIT_BEATS="beats", + SENML_UNIT_SIEMENS_PER_METER="S/m", +) diff --git a/micropython/uaiohttpclient/README b/micropython/uaiohttpclient/README index a3d88b0a4..1222f9d61 100644 --- a/micropython/uaiohttpclient/README +++ b/micropython/uaiohttpclient/README @@ -1,4 +1,4 @@ -uaiohttpclient is an HTTP client module for MicroPython uasyncio module, +uaiohttpclient is an HTTP client module for MicroPython asyncio module, with API roughly compatible with aiohttp (https://github.com/KeepSafe/aiohttp) module. Note that only client is implemented, for server see picoweb microframework. diff --git a/micropython/uaiohttpclient/example.py b/micropython/uaiohttpclient/example.py index 4134c7ee7..540d1b3de 100644 --- a/micropython/uaiohttpclient/example.py +++ b/micropython/uaiohttpclient/example.py @@ -1,31 +1,16 @@ # # uaiohttpclient - fetch URL passed as command line argument. # -import uasyncio as asyncio +import sys +import asyncio import uaiohttpclient as aiohttp -def print_stream(resp): - print((yield from resp.read())) - return - while True: - line = yield from reader.readline() - if not line: - break - print(line.rstrip()) - - -def run(url): - resp = yield from aiohttp.request("GET", url) +async def run(url): + resp = await aiohttp.request("GET", url) print(resp) - yield from print_stream(resp) + print(await resp.read()) -import sys -import logging - -logging.basicConfig(level=logging.INFO) url = sys.argv[1] -loop = asyncio.get_event_loop() -loop.run_until_complete(run(url)) -loop.close() +asyncio.run(run(url)) diff --git a/micropython/uaiohttpclient/manifest.py b/micropython/uaiohttpclient/manifest.py index 72dd9671c..8b35e0a70 100644 --- a/micropython/uaiohttpclient/manifest.py +++ b/micropython/uaiohttpclient/manifest.py @@ -1,4 +1,4 @@ -metadata(description="HTTP client module for MicroPython uasyncio module", version="0.5.1") +metadata(description="HTTP client module for MicroPython asyncio module", version="0.5.2") # Originally written by Paul Sokolovsky. diff --git a/micropython/uaiohttpclient/uaiohttpclient.py b/micropython/uaiohttpclient/uaiohttpclient.py index 25b2e62a9..2e782638c 100644 --- a/micropython/uaiohttpclient/uaiohttpclient.py +++ b/micropython/uaiohttpclient/uaiohttpclient.py @@ -1,12 +1,12 @@ -import uasyncio as asyncio +import asyncio class ClientResponse: def __init__(self, reader): self.content = reader - def read(self, sz=-1): - return (yield from self.content.read(sz)) + async def read(self, sz=-1): + return await self.content.read(sz) def __repr__(self): return "" % (self.status, self.headers) @@ -17,22 +17,22 @@ def __init__(self, reader): self.content = reader self.chunk_size = 0 - def read(self, sz=4 * 1024 * 1024): + async def read(self, sz=4 * 1024 * 1024): if self.chunk_size == 0: - l = yield from self.content.readline() + line = await self.content.readline() # print("chunk line:", l) - l = l.split(b";", 1)[0] - self.chunk_size = int(l, 16) + line = line.split(b";", 1)[0] + self.chunk_size = int(line, 16) # print("chunk size:", self.chunk_size) if self.chunk_size == 0: # End of message - sep = yield from self.content.read(2) + sep = await self.content.read(2) assert sep == b"\r\n" return b"" - data = yield from self.content.read(min(sz, self.chunk_size)) + data = await self.content.read(min(sz, self.chunk_size)) self.chunk_size -= len(data) if self.chunk_size == 0: - sep = yield from self.content.read(2) + sep = await self.content.read(2) assert sep == b"\r\n" return data @@ -40,40 +40,46 @@ def __repr__(self): return "" % (self.status, self.headers) -def request_raw(method, url): +async def request_raw(method, url): try: proto, dummy, host, path = url.split("/", 3) except ValueError: proto, dummy, host = url.split("/", 2) path = "" + + if ":" in host: + host, port = host.split(":") + port = int(port) + else: + port = 80 + if proto != "http:": raise ValueError("Unsupported protocol: " + proto) - reader, writer = yield from asyncio.open_connection(host, 80) - # Use protocol 1.0, because 1.1 always allows to use chunked transfer-encoding - # But explicitly set Connection: close, even though this should be default for 1.0, - # because some servers misbehave w/o it. + reader, writer = await asyncio.open_connection(host, port) + # Use protocol 1.0, because 1.1 always allows to use chunked + # transfer-encoding But explicitly set Connection: close, even + # though this should be default for 1.0, because some servers + # misbehave w/o it. query = "%s /%s HTTP/1.0\r\nHost: %s\r\nConnection: close\r\nUser-Agent: compat\r\n\r\n" % ( method, path, host, ) - yield from writer.awrite(query.encode("latin-1")) - # yield from writer.aclose() + await writer.awrite(query.encode("latin-1")) return reader -def request(method, url): +async def request(method, url): redir_cnt = 0 - redir_url = None while redir_cnt < 2: - reader = yield from request_raw(method, url) + reader = await request_raw(method, url) headers = [] - sline = yield from reader.readline() + sline = await reader.readline() sline = sline.split(None, 2) status = int(sline[1]) chunked = False while True: - line = yield from reader.readline() + line = await reader.readline() if not line or line == b"\r\n": break headers.append(line) @@ -85,7 +91,7 @@ def request(method, url): if 301 <= status <= 303: redir_cnt += 1 - yield from reader.aclose() + await reader.aclose() continue break diff --git a/micropython/ucontextlib/tests.py b/micropython/ucontextlib/tests.py index 4fd026ae7..163175d82 100644 --- a/micropython/ucontextlib/tests.py +++ b/micropython/ucontextlib/tests.py @@ -24,7 +24,7 @@ def test_context_manager(self): def test_context_manager_on_error(self): exc = Exception() try: - with self._manager(123) as x: + with self._manager(123): raise exc except Exception as e: self.assertEqual(exc, e) diff --git a/micropython/udnspkt/example_resolve.py b/micropython/udnspkt/example_resolve.py index c1215045a..d72c17a48 100644 --- a/micropython/udnspkt/example_resolve.py +++ b/micropython/udnspkt/example_resolve.py @@ -1,15 +1,15 @@ -import uio -import usocket +import io +import socket import udnspkt -s = usocket.socket(usocket.AF_INET, usocket.SOCK_DGRAM) -dns_addr = usocket.getaddrinfo("127.0.0.1", 53)[0][-1] +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +dns_addr = socket.getaddrinfo("127.0.0.1", 53)[0][-1] def resolve(domain, is_ipv6): - buf = uio.BytesIO(48) + buf = io.BytesIO(48) udnspkt.make_req(buf, "google.com", is_ipv6) v = buf.getvalue() print("query: ", v) @@ -17,11 +17,11 @@ def resolve(domain, is_ipv6): resp = s.recv(1024) print("resp:", resp) - buf = uio.BytesIO(resp) + buf = io.BytesIO(resp) addr = udnspkt.parse_resp(buf, is_ipv6) print("bin addr:", addr) - print("addr:", usocket.inet_ntop(usocket.AF_INET6 if is_ipv6 else usocket.AF_INET, addr)) + print("addr:", socket.inet_ntop(socket.AF_INET6 if is_ipv6 else socket.AF_INET, addr)) resolve("google.com", False) diff --git a/micropython/udnspkt/manifest.py b/micropython/udnspkt/manifest.py index 382b8dd74..2c2a78d2b 100644 --- a/micropython/udnspkt/manifest.py +++ b/micropython/udnspkt/manifest.py @@ -1,4 +1,4 @@ -metadata(description="Make and parse DNS packets (Sans I/O approach).", version="0.1") +metadata(description="Make and parse DNS packets (Sans I/O approach).", version="0.1.0") # Originally written by Paul Sokolovsky. diff --git a/micropython/udnspkt/udnspkt.py b/micropython/udnspkt/udnspkt.py index e55285975..f3b998a8a 100644 --- a/micropython/udnspkt/udnspkt.py +++ b/micropython/udnspkt/udnspkt.py @@ -1,6 +1,3 @@ -import uio - - def write_fqdn(buf, name): parts = name.split(".") for p in parts: @@ -43,36 +40,28 @@ def parse_resp(buf, is_ipv6): if is_ipv6: typ = 28 # AAAA - id = buf.readbin(">H") + buf.readbin(">H") # id flags = buf.readbin(">H") assert flags & 0x8000 - qcnt = buf.readbin(">H") + buf.readbin(">H") # qcnt acnt = buf.readbin(">H") - nscnt = buf.readbin(">H") - addcnt = buf.readbin(">H") - # print(qcnt, acnt, nscnt, addcnt) + buf.readbin(">H") # nscnt + buf.readbin(">H") # addcnt skip_fqdn(buf) - v = buf.readbin(">H") - # print(v) - v = buf.readbin(">H") - # print(v) + buf.readbin(">H") + buf.readbin(">H") for i in range(acnt): # print("Resp #%d" % i) # v = read_fqdn(buf) # print(v) skip_fqdn(buf) - t = buf.readbin(">H") - # print("Type", t) - v = buf.readbin(">H") - # print("Class", v) - v = buf.readbin(">I") - # print("TTL", v) + t = buf.readbin(">H") # Type + buf.readbin(">H") # Class + buf.readbin(">I") # TTL rlen = buf.readbin(">H") - # print("rlen", rlen) rval = buf.read(rlen) - # print(rval) if t == typ: return rval diff --git a/micropython/umqtt.robust/example_sub_robust.py b/micropython/umqtt.robust/example_sub_robust.py index c991c70a1..f09befe02 100644 --- a/micropython/umqtt.robust/example_sub_robust.py +++ b/micropython/umqtt.robust/example_sub_robust.py @@ -19,8 +19,7 @@ def sub_cb(topic, msg): # # There can be a problem when a session for a given client exists, # but doesn't have subscriptions a particular application expects. -# In this case, a session needs to be cleaned first. See -# example_reset_session.py for an obvious way how to do that. +# In this case, a session needs to be cleaned first. # # In an actual application, it's up to its developer how to # manage these issues. One extreme is to have external "provisioning" diff --git a/micropython/umqtt.robust/umqtt/robust.py b/micropython/umqtt.robust/umqtt/robust.py index 4cc10e336..51596de9e 100644 --- a/micropython/umqtt.robust/umqtt/robust.py +++ b/micropython/umqtt.robust/umqtt/robust.py @@ -1,4 +1,4 @@ -import utime +import time from . import simple @@ -7,7 +7,7 @@ class MQTTClient(simple.MQTTClient): DEBUG = False def delay(self, i): - utime.sleep(self.DELAY) + time.sleep(self.DELAY) def log(self, in_reconnect, e): if self.DEBUG: diff --git a/micropython/umqtt.simple/example_pub_button.py b/micropython/umqtt.simple/example_pub_button.py index 1bc47bc5e..2a3ec851e 100644 --- a/micropython/umqtt.simple/example_pub_button.py +++ b/micropython/umqtt.simple/example_pub_button.py @@ -1,5 +1,5 @@ import time -import ubinascii +import binascii import machine from umqtt.simple import MQTTClient from machine import Pin @@ -10,7 +10,7 @@ # Default MQTT server to connect to SERVER = "192.168.1.35" -CLIENT_ID = ubinascii.hexlify(machine.unique_id()) +CLIENT_ID = binascii.hexlify(machine.unique_id()) TOPIC = b"led" diff --git a/micropython/umqtt.simple/example_sub_led.py b/micropython/umqtt.simple/example_sub_led.py index 73c6b58d8..c3dcf08d2 100644 --- a/micropython/umqtt.simple/example_sub_led.py +++ b/micropython/umqtt.simple/example_sub_led.py @@ -1,6 +1,6 @@ from umqtt.simple import MQTTClient from machine import Pin -import ubinascii +import binascii import machine import micropython @@ -11,7 +11,7 @@ # Default MQTT server to connect to SERVER = "192.168.1.35" -CLIENT_ID = ubinascii.hexlify(machine.unique_id()) +CLIENT_ID = binascii.hexlify(machine.unique_id()) TOPIC = b"led" diff --git a/micropython/umqtt.simple/manifest.py b/micropython/umqtt.simple/manifest.py index 19617a5ee..709a27505 100644 --- a/micropython/umqtt.simple/manifest.py +++ b/micropython/umqtt.simple/manifest.py @@ -1,5 +1,7 @@ -metadata(description="Lightweight MQTT client for MicroPython.", version="1.3.4") +metadata(description="Lightweight MQTT client for MicroPython.", version="1.6.0") # Originally written by Paul Sokolovsky. +require("ssl") + package("umqtt") diff --git a/micropython/umqtt.simple/umqtt/simple.py b/micropython/umqtt.simple/umqtt/simple.py index 2b269473b..d9cdffc47 100644 --- a/micropython/umqtt.simple/umqtt/simple.py +++ b/micropython/umqtt.simple/umqtt/simple.py @@ -1,6 +1,6 @@ -import usocket as socket -import ustruct as struct -from ubinascii import hexlify +import socket +import struct +from binascii import hexlify class MQTTException(Exception): @@ -16,7 +16,7 @@ def __init__( user=None, password=None, keepalive=0, - ssl=False, + ssl=None, ssl_params={}, ): if port == 0: @@ -62,20 +62,24 @@ def set_last_will(self, topic, msg, retain=False, qos=0): self.lw_qos = qos self.lw_retain = retain - def connect(self, clean_session=True): + def connect(self, clean_session=True, timeout=None): self.sock = socket.socket() + self.sock.settimeout(timeout) addr = socket.getaddrinfo(self.server, self.port)[0][-1] self.sock.connect(addr) - if self.ssl: - import ussl + if self.ssl is True: + # Legacy support for ssl=True and ssl_params arguments. + import ssl - self.sock = ussl.wrap_socket(self.sock, **self.ssl_params) + self.sock = ssl.wrap_socket(self.sock, **self.ssl_params) + elif self.ssl: + self.sock = self.ssl.wrap_socket(self.sock, server_hostname=self.server) premsg = bytearray(b"\x10\0\0\0\0\0") msg = bytearray(b"\x04MQTT\x04\x02\0\0") sz = 10 + 2 + len(self.client_id) msg[6] = clean_session << 1 - if self.user is not None: + if self.user: sz += 2 + len(self.user) + 2 + len(self.pswd) msg[6] |= 0xC0 if self.keepalive: @@ -101,7 +105,7 @@ def connect(self, clean_session=True): if self.lw_topic: self._send_str(self.lw_topic) self._send_str(self.lw_msg) - if self.user is not None: + if self.user: self._send_str(self.user) self._send_str(self.pswd) resp = self.sock.read(4) diff --git a/micropython/urequests/README.md b/micropython/urequests/README.md new file mode 100644 index 000000000..f6612b356 --- /dev/null +++ b/micropython/urequests/README.md @@ -0,0 +1,9 @@ +## urequests compatibility + +The MicroPython version of +[requests](https://requests.readthedocs.io/en/latest/) was previously called +`urequests` and a lot of existing code depends on being able to still +import the module by that name. + +This package provides a wrapper to allow this. Prefer to install and use the +`requests` package instead. diff --git a/micropython/urequests/manifest.py b/micropython/urequests/manifest.py new file mode 100644 index 000000000..3fbe61c25 --- /dev/null +++ b/micropython/urequests/manifest.py @@ -0,0 +1,5 @@ +metadata(version="0.8.0", pypi="requests") + +require("requests") + +module("urequests.py") diff --git a/micropython/urequests/urequests.py b/micropython/urequests/urequests.py new file mode 100644 index 000000000..227a1ae5c --- /dev/null +++ b/micropython/urequests/urequests.py @@ -0,0 +1,8 @@ +# This module provides a backwards-compatble import for `urequests`. +# It lazy-loads from `requests` without duplicating its globals dict. + + +def __getattr__(attr): + import requests + + return getattr(requests, attr) diff --git a/micropython/urllib.urequest/manifest.py b/micropython/urllib.urequest/manifest.py index cb4c1569c..2790208a7 100644 --- a/micropython/urllib.urequest/manifest.py +++ b/micropython/urllib.urequest/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.6") +metadata(version="0.7.0") # Originally written by Paul Sokolovsky. diff --git a/micropython/urllib.urequest/urllib/urequest.py b/micropython/urllib.urequest/urllib/urequest.py index 4c654d45e..f83cbaa94 100644 --- a/micropython/urllib.urequest/urllib/urequest.py +++ b/micropython/urllib.urequest/urllib/urequest.py @@ -1,4 +1,4 @@ -import usocket +import socket def urlopen(url, data=None, method="GET"): @@ -12,7 +12,7 @@ def urlopen(url, data=None, method="GET"): if proto == "http:": port = 80 elif proto == "https:": - import ussl + import tls port = 443 else: @@ -22,14 +22,16 @@ def urlopen(url, data=None, method="GET"): host, port = host.split(":", 1) port = int(port) - ai = usocket.getaddrinfo(host, port, 0, usocket.SOCK_STREAM) + ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) ai = ai[0] - s = usocket.socket(ai[0], ai[1], ai[2]) + s = socket.socket(ai[0], ai[1], ai[2]) try: s.connect(ai[-1]) if proto == "https:": - s = ussl.wrap_socket(s, server_hostname=host) + context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + context.verify_mode = tls.CERT_NONE + s = context.wrap_socket(s, server_hostname=host) s.write(method) s.write(b" /") @@ -46,10 +48,10 @@ def urlopen(url, data=None, method="GET"): if data: s.write(data) - l = s.readline() - l = l.split(None, 2) + l = s.readline() # Status-Line + # l = l.split(None, 2) # print(l) - status = int(l[1]) + # status = int(l[1]) # FIXME: Status-Code element is not currently checked while True: l = s.readline() if not l or l == b"\r\n": diff --git a/micropython/usb/README.md b/micropython/usb/README.md new file mode 100644 index 000000000..d4b975d12 --- /dev/null +++ b/micropython/usb/README.md @@ -0,0 +1,148 @@ +# USB Drivers + +These packages allow implementing USB functionality on a MicroPython system +using pure Python code. + +Currently only USB device is implemented, not USB host. + +## USB Device support + +### Support + +USB Device support depends on the low-level +[machine.USBDevice](https://docs.micropython.org/en/latest/library/machine.USBDevice.html) +class. This class is new and not supported on all ports, so please check the +documentation for your MicroPython version. It is possible to implement a USB +device using only the low-level USBDevice class. However, the packages here are +higher level and easier to use. + +For more information about how to install packages, or "freeze" them into a +firmware image, consult the [MicroPython documentation on "Package +management"](https://docs.micropython.org/en/latest/reference/packages.html). + +### Examples + +The [examples/device](examples/device) directory in this repo has a range of +examples. After installing necessary packages, you can download an example and +run it with `mpremote run EXAMPLE_FILENAME.py` ([mpremote +docs](https://docs.micropython.org/en/latest/reference/mpremote.html#mpremote-command-run)). + +#### Unexpected serial disconnects + +If you normally connect to your MicroPython device over a USB serial port ("USB +CDC"), then running a USB example will disconnect mpremote when the new USB +device configuration activates and the serial port has to temporarily +disconnect. It is likely that mpremote will print an error. The example should +still start running, if necessary then you can reconnect with mpremote and type +Ctrl-B to restore the MicroPython REPL and/or Ctrl-C to stop the running +example. + +If you use `mpremote run` again while a different USB device configuration is +already active, then the USB serial port may disconnect immediately before the +example runs. This is because mpremote has to soft-reset MicroPython, and when +the existing USB device is reset then the entire USB port needs to reset. If +this happens, run the same `mpremote run` command again. + +We plan to add features to `mpremote` so that this limitation is less +disruptive. Other tools that communicate with MicroPython over the serial port +will encounter similar issues when runtime USB is in use. + +### Initialising runtime USB + +The overall pattern for enabling USB devices at runtime is: + +1. Instantiate the Interface objects for your desired USB device. +2. Call `usb.device.get()` to get the singleton object for the high-level USB device. +3. Call `init(...)` to pass the desired interfaces as arguments, plus any custom + keyword arguments to configure the overall device. + +An example, similar to [mouse_example.py](examples/device/mouse_example.py): + +```py + m = usb.device.mouse.MouseInterface() + usb.device.get().init(m, builtin_driver=True) +``` + +Setting `builtin_driver=True` means that any built-in USB serial port will still +be available. Otherwise, you may permanently lose access to MicroPython until +the next time the device resets. + +See [Unexpected serial disconnects](#Unexpected-serial-disconnects), above, for +an explanation of possible errors or disconnects when the runtime USB device +initialises. + +Placing the call to `usb.device.get().init()` into the `boot.py` of the +MicroPython file system allows the runtime USB device to initialise immediately +on boot, before any built-in USB. This is a feature (not a bug) and allows you +full control over the USB device, for example to only enable USB HID and prevent +REPL access to the system. + +However, note that calling this function on boot without `builtin_driver=True` +will make the MicroPython USB serial interface permanently inaccessible until +you "safe mode boot" (on supported boards) or completely erase the flash of your +device. + +### Package usb-device + +This base package contains the common implementation components for the other +packages, and can be used to implement new and different USB interface support. +All of the other `usb-device-` packages depend on this package, and it +will be automatically installed as needed. + +Specicially, this package provides the `usb.device.get()` function for accessing +the Device singleton object, and the `usb.device.core` module which contains the +low-level classes and utility functions for implementing new USB interface +drivers in Python. The best examples of how to use the core classes is the +source code of the other USB device packages. + +### Package usb-device-keyboard + +This package provides the `usb.device.keyboard` module. See +[keyboard_example.py](examples/device/keyboard_example.py) for an example +program. + +### Package usb-device-mouse + +This package provides the `usb.device.mouse` module. See +[mouse_example.py](examples/device/mouse_example.py) for an example program. + +### Package usb-device-hid + +This package provides the `usb.device.hid` module. USB HID (Human Interface +Device) class allows creating a wide variety of device types. The most common +are mouse and keyboard, which have their own packages in micropython-lib. +However, using the usb-device-hid package directly allows creation of any kind +of HID device. + +See [hid_custom_keypad_example.py](examples/device/hid_custom_keypad_example.py) +for an example of a Keypad HID device with a custom HID descriptor. + +### Package usb-device-cdc + +This package provides the `usb.device.cdc` module. USB CDC (Communications +Device Class) is most commonly used for virtual serial port USB interfaces, and +that is what is supported here. + +The example [cdc_repl_example.py](examples/device/cdc_repl_example.py) +demonstrates how to add a second USB serial interface and duplicate the +MicroPython REPL between the two. + +### Package usb-device-midi + +This package provides the `usb.device.midi` module. This allows implementing +USB MIDI devices in MicroPython. + +The example [midi_example.py](examples/device/midi_example.py) demonstrates how +to create a simple MIDI device to send MIDI data to and from the USB host. + +### Limitations + +#### Buffer thread safety + +The internal Buffer class that's used by most of the USB device classes expects data +to be written to it (i.e. sent to the host) by only one thread. Bytes may be +lost from the USB transfers if more than one thread (or a thread and a callback) +try to write to the buffer simultaneously. + +If writing USB data from multiple sources, your code may need to add +synchronisation (i.e. locks). diff --git a/micropython/usb/examples/device/cdc_repl_example.py b/micropython/usb/examples/device/cdc_repl_example.py new file mode 100644 index 000000000..06dc9a76c --- /dev/null +++ b/micropython/usb/examples/device/cdc_repl_example.py @@ -0,0 +1,47 @@ +# MicroPython USB CDC REPL example +# +# Example demonstrating how to use os.dupterm() to provide the +# MicroPython REPL on a dynamic CDCInterface() serial port. +# +# To run this example: +# +# 1. Make sure `usb-device-cdc` is installed via: mpremote mip install usb-device-cdc +# +# 2. Run the example via: mpremote run cdc_repl_example.py +# +# 3. mpremote will exit with an error after the previous step, because when the +# example runs the existing USB device disconnects and then re-enumerates with +# the second serial port. If you check (for example by running mpremote connect +# list) then you should now see two USB serial devices. +# +# 4. Connect to one of the new ports: mpremote connect PORTNAME +# +# It may be necessary to type Ctrl-B to exit the raw REPL mode and resume the +# interactive REPL after mpremote connects. +# +# MIT license; Copyright (c) 2023-2024 Angus Gratton +import os +import time +import usb.device +from usb.device.cdc import CDCInterface + +cdc = CDCInterface() +cdc.init(timeout=0) # zero timeout makes this non-blocking, suitable for os.dupterm() + +# pass builtin_driver=True so that we get the built-in USB-CDC alongside, +# if it's available. +usb.device.get().init(cdc, builtin_driver=True) + +print("Waiting for USB host to configure the interface...") + +# wait for host enumerate as a CDC device... +while not cdc.is_open(): + time.sleep_ms(100) + +# Note: This example doesn't wait for the host to access the new CDC port, +# which could be done by polling cdc.dtr, as this will block the REPL +# from resuming while this code is still executing. + +print("CDC port enumerated, duplicating REPL...") + +old_term = os.dupterm(cdc) diff --git a/micropython/usb/examples/device/hid_custom_keypad_example.py b/micropython/usb/examples/device/hid_custom_keypad_example.py new file mode 100644 index 000000000..9d427cf10 --- /dev/null +++ b/micropython/usb/examples/device/hid_custom_keypad_example.py @@ -0,0 +1,144 @@ +# MicroPython USB HID custom Keypad example +# +# This example demonstrates creating a custom HID device with its own +# HID descriptor, in this case for a USB number keypad. +# +# For higher level examples that require less code to use, see mouse_example.py +# and keyboard_example.py +# +# To run this example: +# +# 1. Make sure `usb-device-hid` is installed via: mpremote mip install usb-device-hid +# +# 2. Run the example via: mpremote run hid_custom_keypad_example.py +# +# 3. mpremote will exit with an error after the previous step, because when the +# example runs the existing USB device disconnects and then re-enumerates with +# the custom HID interface present. At this point, the example is running. +# +# 4. To see output from the example, re-connect: mpremote connect PORTNAME +# +# MIT license; Copyright (c) 2023 Dave Wickham, 2023-2024 Angus Gratton +from micropython import const +import time +import usb.device +from usb.device.hid import HIDInterface + +_INTERFACE_PROTOCOL_KEYBOARD = const(0x01) + + +def keypad_example(): + k = KeypadInterface() + + usb.device.get().init(k, builtin_driver=True) + + while not k.is_open(): + time.sleep_ms(100) + + while True: + time.sleep(2) + print("Press NumLock...") + k.send_key("") + time.sleep_ms(100) + k.send_key() + time.sleep(1) + # continue + print("Press ...") + for _ in range(3): + time.sleep(0.1) + k.send_key(".") + time.sleep(0.1) + k.send_key() + print("Starting again...") + + +class KeypadInterface(HIDInterface): + # Very basic synchronous USB keypad HID interface + + def __init__(self): + super().__init__( + _KEYPAD_REPORT_DESC, + set_report_buf=bytearray(1), + protocol=_INTERFACE_PROTOCOL_KEYBOARD, + interface_str="MicroPython Keypad", + ) + self.numlock = False + + def on_set_report(self, report_data, _report_id, _report_type): + report = report_data[0] + b = bool(report & 1) + if b != self.numlock: + print("Numlock: ", b) + self.numlock = b + + def send_key(self, key=None): + if key is None: + self.send_report(b"\x00") + else: + self.send_report(_key_to_id(key).to_bytes(1, "big")) + + +# See HID Usages and Descriptions 1.4, section 10 Keyboard/Keypad Page (0x07) +# +# This keypad example has a contiguous series of keys (KEYPAD_KEY_IDS) starting +# from the NumLock/Clear keypad key (0x53), but you can send any Key IDs from +# the table in the HID Usages specification. +_KEYPAD_KEY_OFFS = const(0x53) + +_KEYPAD_KEY_IDS = [ + "", + "/", + "*", + "-", + "+", + "", + "1", + "2", + "3", + "4", + "5", + "6", + "7", + "8", + "9", + "0", + ".", +] + + +def _key_to_id(key): + # This is a little slower than making a dict for lookup, but uses + # less memory and O(n) can be fast enough when n is small. + return _KEYPAD_KEY_IDS.index(key) + _KEYPAD_KEY_OFFS + + +# HID Report descriptor for a numeric keypad +# +# fmt: off +_KEYPAD_REPORT_DESC = ( + b'\x05\x01' # Usage Page (Generic Desktop) + b'\x09\x07' # Usage (Keypad) + b'\xA1\x01' # Collection (Application) + b'\x05\x07' # Usage Page (Keypad) + b'\x19\x00' # Usage Minimum (0) + b'\x29\xFF' # Usage Maximum (ff) + b'\x15\x00' # Logical Minimum (0) + b'\x25\xFF' # Logical Maximum (ff) + b'\x95\x01' # Report Count (1), + b'\x75\x08' # Report Size (8), + b'\x81\x00' # Input (Data, Array, Absolute) + b'\x05\x08' # Usage page (LEDs) + b'\x19\x01' # Usage Minimum (1) + b'\x29\x01' # Usage Maximum (1), + b'\x95\x01' # Report Count (1), + b'\x75\x01' # Report Size (1), + b'\x91\x02' # Output (Data, Variable, Absolute) + b'\x95\x01' # Report Count (1), + b'\x75\x07' # Report Size (7), + b'\x91\x01' # Output (Constant) - padding bits + b'\xC0' # End Collection +) +# fmt: on + + +keypad_example() diff --git a/micropython/usb/examples/device/keyboard_example.py b/micropython/usb/examples/device/keyboard_example.py new file mode 100644 index 000000000..d8994ff1b --- /dev/null +++ b/micropython/usb/examples/device/keyboard_example.py @@ -0,0 +1,97 @@ +# MicroPython USB Keyboard example +# +# To run this example: +# +# 1. Check the KEYS assignment below, and connect buttons or switches to the +# assigned GPIOs. You can change the entries as needed, look up the reference +# for your board to see what pins are available. Note that the example uses +# "active low" logic, so pressing a switch or button should switch the +# connected pin to Ground (0V). +# +# 2. Make sure `usb-device-keyboard` is installed via: mpremote mip install usb-device-keyboard +# +# 3. Run the example via: mpremote run keyboard_example.py +# +# 4. mpremote will exit with an error after the previous step, because when the +# example runs the existing USB device disconnects and then re-enumerates with +# the keyboard interface present. At this point, the example is running. +# +# 5. The example doesn't print anything to the serial port, but to stop it first +# re-connect: mpremote connect PORTNAME +# +# 6. Type Ctrl-C to interrupt the running example and stop it. You may have to +# also type Ctrl-B to restore the interactive REPL. +# +# To implement a keyboard with different USB HID characteristics, copy the +# usb-device-keyboard/usb/device/keyboard.py file into your own project and modify +# KeyboardInterface. +# +# MIT license; Copyright (c) 2024 Angus Gratton +import usb.device +from usb.device.keyboard import KeyboardInterface, KeyCode, LEDCode +from machine import Pin +import time + +# Tuples mapping Pin inputs to the KeyCode each input generates +# +# (Big keyboards usually multiplex multiple keys per input with a scan matrix, +# but this is a simple example.) +KEYS = ( + (Pin.cpu.GPIO10, KeyCode.CAPS_LOCK), + (Pin.cpu.GPIO11, KeyCode.LEFT_SHIFT), + (Pin.cpu.GPIO12, KeyCode.M), + (Pin.cpu.GPIO13, KeyCode.P), + # ... add more pin to KeyCode mappings here if needed +) + +# Tuples mapping Pin outputs to the LEDCode that turns the output on +LEDS = ( + (Pin.board.LED, LEDCode.CAPS_LOCK), + # ... add more pin to LEDCode mappings here if needed +) + + +class ExampleKeyboard(KeyboardInterface): + def on_led_update(self, led_mask): + # print(hex(led_mask)) + for pin, code in LEDS: + # Set the pin high if 'code' bit is set in led_mask + pin(code & led_mask) + + +def keyboard_example(): + # Initialise all the pins as active-low inputs with pullup resistors + for pin, _ in KEYS: + pin.init(Pin.IN, Pin.PULL_UP) + + # Initialise all the LEDs as active-high outputs + for pin, _ in LEDS: + pin.init(Pin.OUT, value=0) + + # Register the keyboard interface and re-enumerate + k = ExampleKeyboard() + usb.device.get().init(k, builtin_driver=True) + + print("Entering keyboard loop...") + + keys = [] # Keys held down, reuse the same list object + prev_keys = [None] # Previous keys, starts with a dummy value so first + # iteration will always send + while True: + if k.is_open(): + keys.clear() + for pin, code in KEYS: + if not pin(): # active-low + keys.append(code) + if keys != prev_keys: + # print(keys) + k.send_keys(keys) + prev_keys.clear() + prev_keys.extend(keys) + + # This simple example scans each input in an infinite loop, but a more + # complex implementation would probably use a timer or similar. + time.sleep_ms(1) + + +keyboard_example() diff --git a/micropython/usb/examples/device/midi_example.py b/micropython/usb/examples/device/midi_example.py new file mode 100644 index 000000000..55fe8af69 --- /dev/null +++ b/micropython/usb/examples/device/midi_example.py @@ -0,0 +1,78 @@ +# MicroPython USB MIDI example +# +# This example demonstrates creating a custom MIDI device. +# +# To run this example: +# +# 1. Make sure `usb-device-midi` is installed via: mpremote mip install usb-device-midi +# +# 2. Run the example via: mpremote run midi_example.py +# +# 3. mpremote will exit with an error after the previous step, because when the +# example runs the existing USB device disconnects and then re-enumerates with +# the MIDI interface present. At this point, the example is running. +# +# 4. To see output from the example, re-connect: mpremote connect PORTNAME +# +# +# MIT license; Copyright (c) 2023-2024 Angus Gratton +import usb.device +from usb.device.midi import MIDIInterface +import time + + +class MIDIExample(MIDIInterface): + # Very simple example event handler functions, showing how to receive note + # and control change messages sent from the host to the device. + # + # If you need to send MIDI data to the host, then it's fine to instantiate + # MIDIInterface class directly. + + def on_open(self): + super().on_open() + print("Device opened by host") + + def on_note_on(self, channel, pitch, vel): + print(f"RX Note On channel {channel} pitch {pitch} velocity {vel}") + + def on_note_off(self, channel, pitch, vel): + print(f"RX Note Off channel {channel} pitch {pitch} velocity {vel}") + + def on_control_change(self, channel, controller, value): + print(f"RX Control channel {channel} controller {controller} value {value}") + + +m = MIDIExample() +# Remove builtin_driver=True if you don't want the MicroPython serial REPL available. +usb.device.get().init(m, builtin_driver=True) + +print("Waiting for USB host to configure the interface...") + +while not m.is_open(): + time.sleep_ms(100) + +print("Starting MIDI loop...") + +# TX constants +CHANNEL = 0 +PITCH = 60 +CONTROLLER = 64 + +control_val = 0 + +while m.is_open(): + time.sleep(1) + print(f"TX Note On channel {CHANNEL} pitch {PITCH}") + m.note_on(CHANNEL, PITCH) # Velocity is an optional third argument + time.sleep(0.5) + print(f"TX Note Off channel {CHANNEL} pitch {PITCH}") + m.note_off(CHANNEL, PITCH) + time.sleep(1) + print(f"TX Control channel {CHANNEL} controller {CONTROLLER} value {control_val}") + m.control_change(CHANNEL, CONTROLLER, control_val) + control_val += 1 + if control_val == 0x7F: + control_val = 0 + time.sleep(1) + +print("USB host has reset device, example done.") diff --git a/micropython/usb/examples/device/mouse_example.py b/micropython/usb/examples/device/mouse_example.py new file mode 100644 index 000000000..c73d6cfa6 --- /dev/null +++ b/micropython/usb/examples/device/mouse_example.py @@ -0,0 +1,52 @@ +# MicroPython USB Mouse example +# +# To run this example: +# +# 1. Make sure `usb-device-mouse` is installed via: mpremote mip install usb-device-mouse +# +# 2. Run the example via: mpremote run mouse_example.py +# +# 3. mpremote will exit with an error after the previous step, because when the +# example runs the existing USB device disconnects and then re-enumerates with +# the mouse interface present. At this point, the example is running. +# +# 4. You should see the mouse move and right click. At this point, the example +# is finished executing. +# +# To implement a more complex mouse with more buttons or other custom interface +# features, copy the usb-device-mouse/usb/device/mouse.py file into your own +# project and modify MouseInterface. +# +# MIT license; Copyright (c) 2023-2024 Angus Gratton +import time +import usb.device +from usb.device.mouse import MouseInterface + + +def mouse_example(): + m = MouseInterface() + + # Note: builtin_driver=True means that if there's a USB-CDC REPL + # available then it will appear as well as the HID device. + usb.device.get().init(m, builtin_driver=True) + + # wait for host to enumerate as a HID device... + while not m.is_open(): + time.sleep_ms(100) + + time.sleep_ms(2000) + + print("Moving...") + m.move_by(-100, 0) + m.move_by(-100, 0) + time.sleep_ms(500) + + print("Clicking...") + m.click_right(True) + time.sleep_ms(200) + m.click_right(False) + + print("Done!") + + +mouse_example() diff --git a/micropython/usb/usb-device-cdc/manifest.py b/micropython/usb/usb-device-cdc/manifest.py new file mode 100644 index 000000000..e844b6f01 --- /dev/null +++ b/micropython/usb/usb-device-cdc/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.2") +require("usb-device") +package("usb") diff --git a/micropython/usb/usb-device-cdc/usb/device/cdc.py b/micropython/usb/usb-device-cdc/usb/device/cdc.py new file mode 100644 index 000000000..0acea184f --- /dev/null +++ b/micropython/usb/usb-device-cdc/usb/device/cdc.py @@ -0,0 +1,444 @@ +# MicroPython USB CDC module +# MIT license; Copyright (c) 2022 Martin Fischer, 2023-2024 Angus Gratton +import io +import time +import errno +import machine +import struct +from micropython import const + +from .core import Interface, Buffer, split_bmRequestType + +_EP_IN_FLAG = const(1 << 7) + +# Control transfer stages +_STAGE_IDLE = const(0) +_STAGE_SETUP = const(1) +_STAGE_DATA = const(2) +_STAGE_ACK = const(3) + +# Request types +_REQ_TYPE_STANDARD = const(0x0) +_REQ_TYPE_CLASS = const(0x1) +_REQ_TYPE_VENDOR = const(0x2) +_REQ_TYPE_RESERVED = const(0x3) + +_DEV_CLASS_MISC = const(0xEF) +_CS_DESC_TYPE = const(0x24) # CS Interface type communication descriptor + +# CDC control interface definitions +_INTERFACE_CLASS_CDC = const(2) +_INTERFACE_SUBCLASS_CDC = const(2) # Abstract Control Mode +_PROTOCOL_NONE = const(0) # no protocol + +# CDC descriptor subtype +# see also CDC120.pdf, table 13 +_CDC_FUNC_DESC_HEADER = const(0) +_CDC_FUNC_DESC_CALL_MANAGEMENT = const(1) +_CDC_FUNC_DESC_ABSTRACT_CONTROL = const(2) +_CDC_FUNC_DESC_UNION = const(6) + +# CDC class requests, table 13, PSTN subclass +_SET_LINE_CODING_REQ = const(0x20) +_GET_LINE_CODING_REQ = const(0x21) +_SET_CONTROL_LINE_STATE = const(0x22) +_SEND_BREAK_REQ = const(0x23) + +_LINE_CODING_STOP_BIT_1 = const(0) +_LINE_CODING_STOP_BIT_1_5 = const(1) +_LINE_CODING_STOP_BIT_2 = const(2) + +_LINE_CODING_PARITY_NONE = const(0) +_LINE_CODING_PARITY_ODD = const(1) +_LINE_CODING_PARITY_EVEN = const(2) +_LINE_CODING_PARITY_MARK = const(3) +_LINE_CODING_PARITY_SPACE = const(4) + +_LINE_STATE_DTR = const(1) +_LINE_STATE_RTS = const(2) + +_PARITY_BITS_REPR = "NOEMS" +_STOP_BITS_REPR = ("1", "1.5", "2") + +# Other definitions +_CDC_VERSION = const(0x0120) # release number in binary-coded decimal + +# Number of endpoints in each interface +_CDC_CONTROL_EP_NUM = const(1) +_CDC_DATA_EP_NUM = const(2) + +# CDC data interface definitions +_CDC_ITF_DATA_CLASS = const(0xA) +_CDC_ITF_DATA_SUBCLASS = const(0) +_CDC_ITF_DATA_PROT = const(0) # no protocol + +# Length of the bulk transfer endpoints. Maybe should be configurable? +_BULK_EP_LEN = const(64) + +# MicroPython error constants (negated as IOBase.ioctl uses negative return values for error codes) +# these must match values in py/mperrno.h +_MP_EINVAL = const(-22) +_MP_ETIMEDOUT = const(-110) + +# MicroPython stream ioctl requests, same as py/stream.h +_MP_STREAM_FLUSH = const(1) +_MP_STREAM_POLL = const(3) + +# MicroPython ioctl poll values, same as py/stream.h +_MP_STREAM_POLL_WR = const(0x04) +_MP_STREAM_POLL_RD = const(0x01) +_MP_STREAM_POLL_HUP = const(0x10) + + +class CDCInterface(io.IOBase, Interface): + # USB CDC serial device class, designed to resemble machine.UART + # with some additional methods. + # + # Relies on multiple inheritance so it can be an io.IOBase for stream + # functions and also a Interface (actually an Interface Association + # Descriptor holding two interfaces.) + def __init__(self, **kwargs): + # io.IOBase has no __init__() + Interface.__init__(self) + + # Callbacks for particular control changes initiated by the host + self.break_cb = None # Host sent a "break" condition + self.line_state_cb = None + self.line_coding_cb = None + + self._line_state = 0 # DTR & RTS + # Set a default line coding of 115200/8N1 + self._line_coding = bytearray(b"\x00\xc2\x01\x00\x00\x00\x08") + + self._wb = () # Optional write Buffer (IN endpoint), set by CDC.init() + self._rb = () # Optional read Buffer (OUT endpoint), set by CDC.init() + self._timeout = 1000 # set from CDC.init() as well + + # one control interface endpoint, two data interface endpoints + self.ep_c_in = self.ep_d_in = self.ep_d_out = None + + self._c_itf = None # Number of control interface, data interface is one more + + self.init(**kwargs) + + def init( + self, baudrate=9600, bits=8, parity="N", stop=1, timeout=None, txbuf=256, rxbuf=256, flow=0 + ): + # Configure the CDC serial port. Note that many of these settings like + # baudrate, bits, parity, stop don't change the USB-CDC device behavior + # at all, only the "line coding" as communicated from/to the USB host. + + # Store initial line coding parameters in the USB CDC binary format + # (there is nothing implemented to further change these from Python + # code, the USB host sets them.) + struct.pack_into( + "= _BULK_EP_LEN): + raise ValueError # Buffer sizes are required, rxbuf must be at least one EP + + self._timeout = timeout + self._wb = Buffer(txbuf) + self._rb = Buffer(rxbuf) + + ### + ### Line State & Line Coding State property getters + ### + + @property + def rts(self): + return bool(self._line_state & _LINE_STATE_RTS) + + @property + def dtr(self): + return bool(self._line_state & _LINE_STATE_DTR) + + # Line Coding Representation + # Byte 0-3 Byte 4 Byte 5 Byte 6 + # dwDTERate bCharFormat bParityType bDataBits + + @property + def baudrate(self): + return struct.unpack("= _BULK_EP_LEN + ): + # Can only submit up to the endpoint length per transaction, otherwise we won't + # get any transfer callback until the full transaction completes. + self.submit_xfer(self.ep_d_out, self._rb.pend_write(_BULK_EP_LEN), self._rd_cb) + + def _rd_cb(self, ep, res, num_bytes): + # Whenever a data OUT transfer ends + if res == 0: + self._rb.finish_write(num_bytes) + self._rd_xfer() + + ### + ### io.IOBase stream implementation + ### + + def write(self, buf): + start = time.ticks_ms() + mv = buf + + while True: + # Keep pushing buf into _wb into it's all gone + nbytes = self._wb.write(mv) + self._wr_xfer() # make sure a transfer is running from _wb + + if nbytes == len(mv): + return len(buf) # Success + + # if buf couldn't be fully written on the first attempt + # convert it to a memoryview to track partial writes + if mv is buf: + mv = memoryview(buf) + mv = mv[nbytes:] + + # check for timeout + if time.ticks_diff(time.ticks_ms(), start) >= self._timeout: + return len(buf) - len(mv) + + machine.idle() + + def read(self, size): + start = time.ticks_ms() + + # Allocate a suitable buffer to read into + if size >= 0: + b = bytearray(size) + else: + # for size == -1, return however many bytes are ready + b = bytearray(self._rb.readable()) + + n = self._readinto(b, start) + if not n: + return None + if n < len(b): + return b[:n] + return b + + def readinto(self, b): + return self._readinto(b, time.ticks_ms()) + + def _readinto(self, b, start): + if len(b) == 0: + return 0 + + n = 0 + m = memoryview(b) + while n < len(b): + # copy out of the read buffer if there is anything available + if self._rb.readable(): + n += self._rb.readinto(m if n == 0 else m[n:]) + self._rd_xfer() # if _rd was previously full, no transfer will be running + if n == len(b): + break # Done, exit before we call machine.idle() + + if time.ticks_diff(time.ticks_ms(), start) >= self._timeout: + break # Timed out + + machine.idle() + + return n or None + + def ioctl(self, req, arg): + if req == _MP_STREAM_POLL: + return ( + (_MP_STREAM_POLL_WR if (arg & _MP_STREAM_POLL_WR) and self._wb.writable() else 0) + | (_MP_STREAM_POLL_RD if (arg & _MP_STREAM_POLL_RD) and self._rb.readable() else 0) + | + # using the USB level "open" (i.e. connected to host) for !HUP, not !DTR (port is open) + (_MP_STREAM_POLL_HUP if (arg & _MP_STREAM_POLL_HUP) and not self.is_open() else 0) + ) + elif req == _MP_STREAM_FLUSH: + start = time.ticks_ms() + # Wait until write buffer contains no bytes for the lower TinyUSB layer to "read" + while self._wb.readable(): + if not self.is_open(): + return _MP_EINVAL + if time.ticks_diff(time.ticks_ms(), start) > self._timeout: + return _MP_ETIMEDOUT + machine.idle() + return 0 + + return _MP_EINVAL + + def flush(self): + # a C implementation of this exists in stream.c, but it's not in io.IOBase + # and can't immediately be called from here (AFAIK) + r = self.ioctl(_MP_STREAM_FLUSH, 0) + if r: + raise OSError(r) diff --git a/micropython/usb/usb-device-hid/manifest.py b/micropython/usb/usb-device-hid/manifest.py new file mode 100644 index 000000000..af9b8cb84 --- /dev/null +++ b/micropython/usb/usb-device-hid/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.0") +require("usb-device") +package("usb") diff --git a/micropython/usb/usb-device-hid/usb/device/hid.py b/micropython/usb/usb-device-hid/usb/device/hid.py new file mode 100644 index 000000000..9e4c70dde --- /dev/null +++ b/micropython/usb/usb-device-hid/usb/device/hid.py @@ -0,0 +1,232 @@ +# MicroPython USB hid module +# +# This implements a base HIDInterface class that can be used directly, +# or subclassed into more specific HID interface types. +# +# MIT license; Copyright (c) 2023 Angus Gratton +from micropython import const +import machine +import struct +import time +from .core import Interface, Descriptor, split_bmRequestType + +_EP_IN_FLAG = const(1 << 7) + +# Control transfer stages +_STAGE_IDLE = const(0) +_STAGE_SETUP = const(1) +_STAGE_DATA = const(2) +_STAGE_ACK = const(3) + +# Request types +_REQ_TYPE_STANDARD = const(0x0) +_REQ_TYPE_CLASS = const(0x1) +_REQ_TYPE_VENDOR = const(0x2) +_REQ_TYPE_RESERVED = const(0x3) + +# Descriptor types +_DESC_HID_TYPE = const(0x21) +_DESC_REPORT_TYPE = const(0x22) +_DESC_PHYSICAL_TYPE = const(0x23) + +# Interface and protocol identifiers +_INTERFACE_CLASS = const(0x03) +_INTERFACE_SUBCLASS_NONE = const(0x00) +_INTERFACE_SUBCLASS_BOOT = const(0x01) + +_INTERFACE_PROTOCOL_NONE = const(0x00) +_INTERFACE_PROTOCOL_KEYBOARD = const(0x01) +_INTERFACE_PROTOCOL_MOUSE = const(0x02) + +# bRequest values for HID control requests +_REQ_CONTROL_GET_REPORT = const(0x01) +_REQ_CONTROL_GET_IDLE = const(0x02) +_REQ_CONTROL_GET_PROTOCOL = const(0x03) +_REQ_CONTROL_GET_DESCRIPTOR = const(0x06) +_REQ_CONTROL_SET_REPORT = const(0x09) +_REQ_CONTROL_SET_IDLE = const(0x0A) +_REQ_CONTROL_SET_PROTOCOL = const(0x0B) + +# Standard descriptor lengths +_STD_DESC_INTERFACE_LEN = const(9) +_STD_DESC_ENDPOINT_LEN = const(7) + + +class HIDInterface(Interface): + # Abstract base class to implement a USB device HID interface in Python. + + def __init__( + self, + report_descriptor, + extra_descriptors=[], + set_report_buf=None, + protocol=_INTERFACE_PROTOCOL_NONE, + interface_str=None, + ): + # Construct a new HID interface. + # + # - report_descriptor is the only mandatory argument, which is the binary + # data consisting of the HID Report Descriptor. See Device Class + # Definition for Human Interface Devices (HID) v1.11 section 6.2.2 Report + # Descriptor, p23. + # + # - extra_descriptors is an optional argument holding additional HID + # descriptors, to append after the mandatory report descriptor. Most + # HID devices do not use these. + # + # - set_report_buf is an optional writable buffer object (i.e. + # bytearray), where SET_REPORT requests from the host can be + # written. Only necessary if the report_descriptor contains Output + # entries. If set, the size must be at least the size of the largest + # Output entry. + # + # - protocol can be set to a specific value as per HID v1.11 section 4.3 Protocols, p9. + # + # - interface_str is an optional string descriptor to associate with the HID USB interface. + super().__init__() + self.report_descriptor = report_descriptor + self.extra_descriptors = extra_descriptors + self._set_report_buf = set_report_buf + self.protocol = protocol + self.interface_str = interface_str + + self._int_ep = None # set during enumeration + + def get_report(self): + return False + + def on_set_report(self, report_data, report_id, report_type): + # Override this function in order to handle SET REPORT requests from the host, + # where it sends data to the HID device. + # + # This function will only be called if the Report descriptor contains at least one Output entry, + # and the set_report_buf argument is provided to the constructor. + # + # Return True to complete the control transfer normally, False to abort it. + return True + + def busy(self): + # Returns True if the interrupt endpoint is busy (i.e. existing transfer is pending) + return self.is_open() and self.xfer_pending(self._int_ep) + + def send_report(self, report_data, timeout_ms=100): + # Helper function to send a HID report in the typical USB interrupt + # endpoint associated with a HID interface. + # + # Returns True if successful, False if HID device is not active or timeout + # is reached without being able to queue the report for sending. + deadline = time.ticks_add(time.ticks_ms(), timeout_ms) + while self.busy(): + if time.ticks_diff(deadline, time.ticks_ms()) <= 0: + return False + machine.idle() + if not self.is_open(): + return False + self.submit_xfer(self._int_ep, report_data) + + def desc_cfg(self, desc, itf_num, ep_num, strs): + # Add the standard interface descriptor + desc.interface( + itf_num, + 1, + _INTERFACE_CLASS, + _INTERFACE_SUBCLASS_NONE, + self.protocol, + len(strs) if self.interface_str else 0, + ) + + if self.interface_str: + strs.append(self.interface_str) + + # As per HID v1.11 section 7.1 Standard Requests, return the contents of + # the standard HID descriptor before the associated endpoint descriptor. + self.get_hid_descriptor(desc) + + # Add the typical single USB interrupt endpoint descriptor associated + # with a HID interface. + self._int_ep = ep_num | _EP_IN_FLAG + desc.endpoint(self._int_ep, "interrupt", 8, 8) + + self.idle_rate = 0 + self.protocol = 0 + + def num_eps(self): + return 1 + + def get_hid_descriptor(self, desc=None): + # Append a full USB HID descriptor from the object's report descriptor + # and optional additional descriptors. + # + # See HID Specification Version 1.1, Section 6.2.1 HID Descriptor p22 + + l = 9 + 3 * len(self.extra_descriptors) # total length + + if desc is None: + desc = Descriptor(bytearray(l)) + + desc.pack( + "> 8 + if desc_type == _DESC_HID_TYPE: + return self.get_hid_descriptor() + if desc_type == _DESC_REPORT_TYPE: + return self.report_descriptor + elif req_type == _REQ_TYPE_CLASS: + # HID Spec p50: 7.2 Class-Specific Requests + if bRequest == _REQ_CONTROL_GET_REPORT: + print("GET_REPORT?") + return False # Unsupported for now + if bRequest == _REQ_CONTROL_GET_IDLE: + return bytes([self.idle_rate]) + if bRequest == _REQ_CONTROL_GET_PROTOCOL: + return bytes([self.protocol]) + if bRequest in (_REQ_CONTROL_SET_IDLE, _REQ_CONTROL_SET_PROTOCOL): + return True + if bRequest == _REQ_CONTROL_SET_REPORT: + return self._set_report_buf # If None, request will stall + return False # Unsupported request + + if stage == _STAGE_ACK: + if req_type == _REQ_TYPE_CLASS: + if bRequest == _REQ_CONTROL_SET_IDLE: + self.idle_rate = wValue >> 8 + elif bRequest == _REQ_CONTROL_SET_PROTOCOL: + self.protocol = wValue + elif bRequest == _REQ_CONTROL_SET_REPORT: + report_id = wValue & 0xFF + report_type = wValue >> 8 + report_data = self._set_report_buf + if wLength < len(report_data): + # need to truncate the response in the callback if we got less bytes + # than allowed for in the buffer + report_data = memoryview(self._set_report_buf)[:wLength] + self.on_set_report(report_data, report_id, report_type) + + return True # allow DATA/ACK stages to complete normally diff --git a/micropython/usb/usb-device-keyboard/manifest.py b/micropython/usb/usb-device-keyboard/manifest.py new file mode 100644 index 000000000..5a2ff307d --- /dev/null +++ b/micropython/usb/usb-device-keyboard/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.1") +require("usb-device-hid") +package("usb") diff --git a/micropython/usb/usb-device-keyboard/usb/device/keyboard.py b/micropython/usb/usb-device-keyboard/usb/device/keyboard.py new file mode 100644 index 000000000..22091c50b --- /dev/null +++ b/micropython/usb/usb-device-keyboard/usb/device/keyboard.py @@ -0,0 +1,233 @@ +# MIT license; Copyright (c) 2023-2024 Angus Gratton +from micropython import const +import time +import usb.device +from usb.device.hid import HIDInterface + +_INTERFACE_PROTOCOL_KEYBOARD = const(0x01) + +_KEY_ARRAY_LEN = const(6) # Size of HID key array, must match report descriptor +_KEY_REPORT_LEN = const(_KEY_ARRAY_LEN + 2) # Modifier Byte + Reserved Byte + Array entries + + +class KeyboardInterface(HIDInterface): + # Synchronous USB keyboard HID interface + + def __init__(self): + super().__init__( + _KEYBOARD_REPORT_DESC, + set_report_buf=bytearray(1), + protocol=_INTERFACE_PROTOCOL_KEYBOARD, + interface_str="MicroPython Keyboard", + ) + self._key_reports = [ + bytearray(_KEY_REPORT_LEN), + bytearray(_KEY_REPORT_LEN), + ] # Ping/pong report buffers + self.numlock = False + + def on_set_report(self, report_data, _report_id, _report_type): + self.on_led_update(report_data[0]) + + def on_led_update(self, led_mask): + # Override to handle keyboard LED updates. led_mask is bitwise ORed + # together values as defined in LEDCode. + pass + + def send_keys(self, down_keys, timeout_ms=100): + # Update the state of the keyboard by sending a report with down_keys + # set, where down_keys is an iterable (list or similar) of integer + # values such as the values defined in KeyCode. + # + # Will block for up to timeout_ms if a previous report is still + # pending to be sent to the host. Returns True on success. + + r, s = self._key_reports # next report buffer to send, spare report buffer + r[0] = 0 # modifier byte + i = 2 # index for next key array item to write to + for k in down_keys: + if k < 0: # Modifier key + r[0] |= -k + elif i < _KEY_REPORT_LEN: + r[i] = k + i += 1 + else: # Excess rollover! Can't report + r[0] = 0 + for i in range(2, _KEY_REPORT_LEN): + r[i] = 0xFF + break + + while i < _KEY_REPORT_LEN: + r[i] = 0 + i += 1 + + if self.send_report(r, timeout_ms): + # Swap buffers if the previous one is newly queued to send, so + # any subsequent call can't modify that buffer mid-send + self._key_reports[0] = s + self._key_reports[1] = r + return True + return False + + +# HID keyboard report descriptor +# +# From p69 of http://www.usb.org/developers/devclass_docs/HID1_11.pdf +# +# fmt: off +_KEYBOARD_REPORT_DESC = ( + b'\x05\x01' # Usage Page (Generic Desktop), + b'\x09\x06' # Usage (Keyboard), + b'\xA1\x01' # Collection (Application), + b'\x05\x07' # Usage Page (Key Codes); + b'\x19\xE0' # Usage Minimum (224), + b'\x29\xE7' # Usage Maximum (231), + b'\x15\x00' # Logical Minimum (0), + b'\x25\x01' # Logical Maximum (1), + b'\x75\x01' # Report Size (1), + b'\x95\x08' # Report Count (8), + b'\x81\x02' # Input (Data, Variable, Absolute), ;Modifier byte + b'\x95\x01' # Report Count (1), + b'\x75\x08' # Report Size (8), + b'\x81\x01' # Input (Constant), ;Reserved byte + b'\x95\x05' # Report Count (5), + b'\x75\x01' # Report Size (1), + b'\x05\x08' # Usage Page (Page# for LEDs), + b'\x19\x01' # Usage Minimum (1), + b'\x29\x05' # Usage Maximum (5), + b'\x91\x02' # Output (Data, Variable, Absolute), ;LED report + b'\x95\x01' # Report Count (1), + b'\x75\x03' # Report Size (3), + b'\x91\x01' # Output (Constant), ;LED report padding + b'\x95\x06' # Report Count (6), + b'\x75\x08' # Report Size (8), + b'\x15\x00' # Logical Minimum (0), + b'\x25\x65' # Logical Maximum(101), + b'\x05\x07' # Usage Page (Key Codes), + b'\x19\x00' # Usage Minimum (0), + b'\x29\x65' # Usage Maximum (101), + b'\x81\x00' # Input (Data, Array), ;Key arrays (6 bytes) + b'\xC0' # End Collection +) +# fmt: on + + +# Standard HID keycodes, as a pseudo-enum class for easy access +# +# Modifier keys are encoded as negative values +class KeyCode: + A = 4 + B = 5 + C = 6 + D = 7 + E = 8 + F = 9 + G = 10 + H = 11 + I = 12 + J = 13 + K = 14 + L = 15 + M = 16 + N = 17 + O = 18 + P = 19 + Q = 20 + R = 21 + S = 22 + T = 23 + U = 24 + V = 25 + W = 26 + X = 27 + Y = 28 + Z = 29 + N1 = 30 # Standard number row keys + N2 = 31 + N3 = 32 + N4 = 33 + N5 = 34 + N6 = 35 + N7 = 36 + N8 = 37 + N9 = 38 + N0 = 39 + ENTER = 40 + ESCAPE = 41 + BACKSPACE = 42 + TAB = 43 + SPACE = 44 + MINUS = 45 # - _ + EQUAL = 46 # = + + OPEN_BRACKET = 47 # [ { + CLOSE_BRACKET = 48 # ] } + BACKSLASH = 49 # \ | + HASH = 50 # # ~ + SEMICOLON = 51 # ; : + QUOTE = 52 # ' " + GRAVE = 53 # ` ~ + COMMA = 54 # , < + DOT = 55 # . > + SLASH = 56 # / ? + CAPS_LOCK = 57 + F1 = 58 + F2 = 59 + F3 = 60 + F4 = 61 + F5 = 62 + F6 = 63 + F7 = 64 + F8 = 65 + F9 = 66 + F10 = 67 + F11 = 68 + F12 = 69 + PRINTSCREEN = 70 + SCROLL_LOCK = 71 + PAUSE = 72 + INSERT = 73 + HOME = 74 + PAGEUP = 75 + DELETE = 76 + END = 77 + PAGEDOWN = 78 + RIGHT = 79 # Arrow keys + LEFT = 80 + DOWN = 81 + UP = 82 + KP_NUM_LOCK = 83 + KP_DIVIDE = 84 + KP_AT = 85 + KP_MULTIPLY = 85 + KP_MINUS = 86 + KP_PLUS = 87 + KP_ENTER = 88 + KP_1 = 89 + KP_2 = 90 + KP_3 = 91 + KP_4 = 92 + KP_5 = 93 + KP_6 = 94 + KP_7 = 95 + KP_8 = 96 + KP_9 = 97 + KP_0 = 98 + + # HID modifier values (negated to allow them to be passed along with the normal keys) + LEFT_CTRL = -0x01 + LEFT_SHIFT = -0x02 + LEFT_ALT = -0x04 + LEFT_UI = -0x08 + RIGHT_CTRL = -0x10 + RIGHT_SHIFT = -0x20 + RIGHT_ALT = -0x40 + RIGHT_UI = -0x80 + + +# HID LED values +class LEDCode: + NUM_LOCK = 0x01 + CAPS_LOCK = 0x02 + SCROLL_LOCK = 0x04 + COMPOSE = 0x08 + KANA = 0x10 diff --git a/micropython/usb/usb-device-midi/manifest.py b/micropython/usb/usb-device-midi/manifest.py new file mode 100644 index 000000000..af9b8cb84 --- /dev/null +++ b/micropython/usb/usb-device-midi/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.0") +require("usb-device") +package("usb") diff --git a/micropython/usb/usb-device-midi/usb/device/midi.py b/micropython/usb/usb-device-midi/usb/device/midi.py new file mode 100644 index 000000000..ecb178ea4 --- /dev/null +++ b/micropython/usb/usb-device-midi/usb/device/midi.py @@ -0,0 +1,306 @@ +# MicroPython USB MIDI module +# MIT license; Copyright (c) 2023 Paul Hamshere, 2023-2024 Angus Gratton +from micropython import const, schedule +import struct + +from .core import Interface, Buffer + +_EP_IN_FLAG = const(1 << 7) + +_INTERFACE_CLASS_AUDIO = const(0x01) +_INTERFACE_SUBCLASS_AUDIO_CONTROL = const(0x01) +_INTERFACE_SUBCLASS_AUDIO_MIDISTREAMING = const(0x03) + +# Audio subclass extends the standard endpoint descriptor +# with two extra bytes +_STD_DESC_AUDIO_ENDPOINT_LEN = const(9) +_CLASS_DESC_ENDPOINT_LEN = const(5) + +_STD_DESC_ENDPOINT_TYPE = const(0x5) + +_JACK_TYPE_EMBEDDED = const(0x01) +_JACK_TYPE_EXTERNAL = const(0x02) + +_JACK_IN_DESC_LEN = const(6) +_JACK_OUT_DESC_LEN = const(9) + +# MIDI Status bytes. For Channel messages these are only the upper 4 bits, ORed with the channel number. +# As per https://www.midi.org/specifications-old/item/table-1-summary-of-midi-message +_MIDI_NOTE_OFF = const(0x80) +_MIDI_NOTE_ON = const(0x90) +_MIDI_POLY_KEYPRESS = const(0xA0) +_MIDI_CONTROL_CHANGE = const(0xB0) + +# USB-MIDI CINs (Code Index Numbers), as per USB MIDI Table 4-1 +_CIN_SYS_COMMON_2BYTE = const(0x2) +_CIN_SYS_COMMON_3BYTE = const(0x3) +_CIN_SYSEX_START = const(0x4) +_CIN_SYSEX_END_1BYTE = const(0x5) +_CIN_SYSEX_END_2BYTE = const(0x6) +_CIN_SYSEX_END_3BYTE = const(0x7) +_CIN_NOTE_OFF = const(0x8) +_CIN_NOTE_ON = const(0x9) +_CIN_POLY_KEYPRESS = const(0xA) +_CIN_CONTROL_CHANGE = const(0xB) +_CIN_PROGRAM_CHANGE = const(0xC) +_CIN_CHANNEL_PRESSURE = const(0xD) +_CIN_PITCH_BEND = const(0xE) +_CIN_SINGLE_BYTE = const(0xF) # Not currently supported + +# Jack IDs for a simple bidrectional MIDI device(!) +_EMB_IN_JACK_ID = const(1) +_EXT_IN_JACK_ID = const(2) +_EMB_OUT_JACK_ID = const(3) +_EXT_OUT_JACK_ID = const(4) + +# Data flows, as modelled by USB-MIDI and this hypothetical interface, are as follows: +# Device RX = USB OUT EP => _EMB_IN_JACK => _EMB_OUT_JACK +# Device TX = _EXT_IN_JACK => _EMB_OUT_JACK => USB IN EP + + +class MIDIInterface(Interface): + # Base class to implement a USB MIDI device in Python. + # + # To be compliant this also regisers a dummy USB Audio interface, but that + # interface isn't otherwise used. + + def __init__(self, rxlen=16, txlen=16): + # Arguments are size of transmit and receive buffers in bytes. + + super().__init__() + self.ep_out = None # Set during enumeration. RX direction (host to device) + self.ep_in = None # TX direction (device to host) + self._rx = Buffer(rxlen) + self._tx = Buffer(txlen) + + # Callbacks for handling received MIDI messages. + # + # Subclasses can choose between overriding on_midi_event + # and handling all MIDI events manually, or overriding the + # functions for note on/off and control change, only. + + def on_midi_event(self, cin, midi0, midi1, midi2): + ch = midi0 & 0x0F + if cin == _CIN_NOTE_ON: + self.on_note_on(ch, midi1, midi2) + elif cin == _CIN_NOTE_OFF: + self.on_note_off(ch, midi1, midi2) + elif cin == _CIN_CONTROL_CHANGE: + self.on_control_change(ch, midi1, midi2) + + def on_note_on(self, channel, pitch, vel): + pass # Override to handle Note On messages + + def on_note_off(self, channel, pitch, vel): + pass # Override to handle Note On messages + + def on_control_change(self, channel, controller, value): + pass # Override to handle Control Change messages + + # Helper functions for sending common MIDI messages + + def note_on(self, channel, pitch, vel=0x40): + self.send_event(_CIN_NOTE_ON, _MIDI_NOTE_ON | channel, pitch, vel) + + def note_off(self, channel, pitch, vel=0x40): + self.send_event(_CIN_NOTE_OFF, _MIDI_NOTE_OFF | channel, pitch, vel) + + def control_change(self, channel, controller, value): + self.send_event(_CIN_CONTROL_CHANGE, _MIDI_CONTROL_CHANGE | channel, controller, value) + + def send_event(self, cin, midi0, midi1=0, midi2=0): + # Queue a MIDI Event Packet to send to the host. + # + # CIN = USB-MIDI Code Index Number, see USB MIDI 1.0 section 4 "USB-MIDI Event Packets" + # + # Remaining arguments are 0-3 MIDI data bytes. + # + # Note this function returns when the MIDI Event Packet has been queued, + # not when it's been received by the host. + # + # Returns False if the TX buffer is full and the MIDI Event could not be queued. + w = self._tx.pend_write() + if len(w) < 4: + return False # TX buffer is full. TODO: block here? + w[0] = cin # leave cable number as 0? + w[1] = midi0 + w[2] = midi1 + w[3] = midi2 + self._tx.finish_write(4) + self._tx_xfer() + return True + + def _tx_xfer(self): + # Keep an active IN transfer to send data to the host, whenever + # there is data to send. + if self.is_open() and not self.xfer_pending(self.ep_in) and self._tx.readable(): + self.submit_xfer(self.ep_in, self._tx.pend_read(), self._tx_cb) + + def _tx_cb(self, ep, res, num_bytes): + if res == 0: + self._tx.finish_read(num_bytes) + self._tx_xfer() + + def _rx_xfer(self): + # Keep an active OUT transfer to receive MIDI events from the host + if self.is_open() and not self.xfer_pending(self.ep_out) and self._rx.writable(): + self.submit_xfer(self.ep_out, self._rx.pend_write(), self._rx_cb) + + def _rx_cb(self, ep, res, num_bytes): + if res == 0: + self._rx.finish_write(num_bytes) + schedule(self._on_rx, None) + self._rx_xfer() + + def on_open(self): + super().on_open() + # kick off any transfers that may have queued while the device was not open + self._tx_xfer() + self._rx_xfer() + + def _on_rx(self, _): + # Receive MIDI events. Called via micropython.schedule, outside of the USB callback function. + m = self._rx.pend_read() + i = 0 + while i <= len(m) - 4: + cin = m[i] & 0x0F + self.on_midi_event(cin, m[i + 1], m[i + 2], m[i + 3]) + i += 4 + self._rx.finish_read(i) + + def desc_cfg(self, desc, itf_num, ep_num, strs): + # Start by registering a USB Audio Control interface, that is required to point to the + # actual MIDI interface + desc.interface(itf_num, 0, _INTERFACE_CLASS_AUDIO, _INTERFACE_SUBCLASS_AUDIO_CONTROL) + + # Append the class-specific AudioControl interface descriptor + desc.pack( + "1 USB interface.) + + def __init__(self): + self._open = False + + def desc_cfg(self, desc, itf_num, ep_num, strs): + # Function to build configuration descriptor contents for this interface + # or group of interfaces. This is called on each interface from + # USBDevice.init(). + # + # This function should insert: + # + # - At least one standard Interface descriptor (can call + # - desc.interface()). + # + # Plus, optionally: + # + # - One or more endpoint descriptors (can call desc.endpoint()). + # - An Interface Association Descriptor, prepended before. + # - Other class-specific configuration descriptor data. + # + # This function is called twice per call to USBDevice.init(). The first + # time the values of all arguments are dummies that are used only to + # calculate the total length of the descriptor. Therefore, anything this + # function does should be idempotent and it should add the same + # descriptors each time. If saving interface numbers or endpoint numbers + # for later + # + # Parameters: + # + # - desc - Descriptor helper to write the configuration descriptor bytes into. + # The first time this function is called 'desc' is a dummy object + # with no backing buffer (exists to count the number of bytes needed). + # + # - itf_num - First bNumInterfaces value to assign. The descriptor + # should contain the same number of interfaces returned by num_itfs(), + # starting from this value. + # + # - ep_num - Address of the first available endpoint number to use for + # endpoint descriptor addresses. Subclasses should save the + # endpoint addresses selected, to look up later (although note the first + # time this function is called, the values will be dummies.) + # + # - strs - list of string descriptors for this USB device. This function + # can append to this list, and then insert the index of the new string + # in the list into the configuration descriptor. + raise NotImplementedError + + def num_itfs(self): + # Return the number of actual USB Interfaces represented by this object + # (as set in desc_cfg().) + # + # Only needs to be overriden if implementing a Interface class that + # represents more than one USB Interface descriptor (i.e. MIDI), or an + # Interface Association Descriptor (i.e. USB-CDC). + return 1 + + def num_eps(self): + # Return the number of USB Endpoint numbers represented by this object + # (as set in desc_cfg().) + # + # Note for each count returned by this function, the interface may + # choose to have both an IN and OUT endpoint (i.e. IN flag is not + # considered a value here.) + # + # This value can be zero, if the USB Host only communicates with this + # interface using control transfers. + return 0 + + def on_open(self): + # Callback called when the USB host accepts the device configuration. + # + # Override this function to initiate any operations that the USB interface + # should do when the USB device is configured to the host. + self._open = True + + def on_reset(self): + # Callback called on every registered interface when the USB device is + # reset by the host. This can happen when the USB device is unplugged, + # or if the host triggers a reset for some other reason. + # + # Override this function to cancel any pending operations specific to + # the interface (outstanding USB transfers are already cancelled). + # + # At this point, no USB functionality is available - on_open() will + # be called later if/when the USB host re-enumerates and configures the + # interface. + self._open = False + + def is_open(self): + # Returns True if the interface has been configured by the host and is in + # active use. + return self._open + + def on_device_control_xfer(self, stage, request): + # Control transfer callback. Override to handle a non-standard device + # control transfer where bmRequestType Recipient is Device, Type is + # utils.REQ_TYPE_CLASS, and the lower byte of wIndex indicates this interface. + # + # (See USB 2.0 specification 9.4 Standard Device Requests, p250). + # + # This particular request type seems pretty uncommon for a device class + # driver to need to handle, most hosts will not send this so most + # implementations won't need to override it. + # + # Parameters: + # + # - stage is one of utils.STAGE_SETUP, utils.STAGE_DATA, utils.STAGE_ACK. + # + # - request is a memoryview into a USB request packet, as per USB 2.0 + # specification 9.3 USB Device Requests, p250. the memoryview is only + # valid while the callback is running. + # + # The function can call split_bmRequestType(request[0]) to split + # bmRequestType into (Recipient, Type, Direction). + # + # Result, any of: + # + # - True to continue the request, False to STALL the endpoint. + # - Buffer interface object to provide a buffer to the host as part of the + # transfer, if applicable. + return False + + def on_interface_control_xfer(self, stage, request): + # Control transfer callback. Override to handle a device control + # transfer where bmRequestType Recipient is Interface, and the lower byte + # of wIndex indicates this interface. + # + # (See USB 2.0 specification 9.4 Standard Device Requests, p250). + # + # bmRequestType Type field may have different values. It's not necessary + # to handle the mandatory Standard requests (bmRequestType Type == + # utils.REQ_TYPE_STANDARD), if the driver returns False in these cases then + # TinyUSB will provide the necessary responses. + # + # See on_device_control_xfer() for a description of the arguments and + # possible return values. + return False + + def on_endpoint_control_xfer(self, stage, request): + # Control transfer callback. Override to handle a device + # control transfer where bmRequestType Recipient is Endpoint and + # the lower byte of wIndex indicates an endpoint address associated + # with this interface. + # + # bmRequestType Type will generally have any value except + # utils.REQ_TYPE_STANDARD, as Standard endpoint requests are handled by + # TinyUSB. The exception is the the Standard "Set Feature" request. This + # is handled by Tiny USB but also passed through to the driver in case it + # needs to change any internal state, but most drivers can ignore and + # return False in this case. + # + # (See USB 2.0 specification 9.4 Standard Device Requests, p250). + # + # See on_device_control_xfer() for a description of the parameters and + # possible return values. + return False + + def xfer_pending(self, ep_addr): + # Return True if a transfer is already pending on ep_addr. + # + # Only one transfer can be submitted at a time. + # + # The transfer is marked pending while a completion callback is running + # for that endpoint, unless this function is called from the callback + # itself. This makes it simple to submit a new transfer from the + # completion callback. + return _dev and _dev._xfer_pending(ep_addr) + + def submit_xfer(self, ep_addr, data, done_cb=None): + # Submit a USB transfer (of any type except control) + # + # Parameters: + # + # - ep_addr. Address of the endpoint to submit the transfer on. Caller is + # responsible for ensuring that ep_addr is correct and belongs to this + # interface. Only one transfer can be active at a time on each endpoint. + # + # - data. Buffer containing data to send, or for data to be read into + # (depending on endpoint direction). + # + # - done_cb. Optional callback function for when the transfer + # completes. The callback is called with arguments (ep_addr, result, + # xferred_bytes) where result is one of xfer_result_t enum (see top of + # this file), and xferred_bytes is an integer. + # + # If the function returns, the transfer is queued. + # + # The function will raise RuntimeError under the following conditions: + # + # - The interface is not "open" (i.e. has not been enumerated and configured + # by the host yet.) + # + # - A transfer is already pending on this endpoint (use xfer_pending() to check + # before sending if needed.) + # + # - A DCD error occurred when queueing the transfer on the hardware. + # + # + # Will raise TypeError if 'data' isn't he correct type of buffer for the + # endpoint transfer direction. + # + # Note that done_cb may be called immediately, possibly before this + # function has returned to the caller. + if not self._open: + raise RuntimeError("Not open") + _dev._submit_xfer(ep_addr, data, done_cb) + + def stall(self, ep_addr, *args): + # Set or get the endpoint STALL state. + # + # To get endpoint stall stage, call with a single argument. + # To set endpoint stall state, call with an additional boolean + # argument to set or clear. + # + # Generally endpoint STALL is handled automatically, but there are some + # device classes that need to explicitly stall or unstall an endpoint + # under certain conditions. + if not self._open or ep_addr not in self._eps: + raise RuntimeError + _dev._usbd.stall(ep_addr, *args) + + +class Descriptor: + # Wrapper class for writing a descriptor in-place into a provided buffer + # + # Doesn't resize the buffer. + # + # Can be initialised with b=None to perform a dummy pass that calculates the + # length needed for the buffer. + def __init__(self, b): + self.b = b + self.o = 0 # offset of data written to the buffer + + def pack(self, fmt, *args): + # Utility function to pack new data into the descriptor + # buffer, starting at the current offset. + # + # Arguments are the same as struct.pack(), but it fills the + # pre-allocated descriptor buffer (growing if needed), instead of + # returning anything. + self.pack_into(fmt, self.o, *args) + + def pack_into(self, fmt, offs, *args): + # Utility function to pack new data into the descriptor at offset 'offs'. + # + # If the data written is before 'offs' then self.o isn't incremented, + # otherwise it's incremented to point at the end of the written data. + end = offs + struct.calcsize(fmt) + if self.b: + struct.pack_into(fmt, self.b, offs, *args) + self.o = max(self.o, end) + + def extend(self, a): + # Extend the descriptor with some bytes-like data + if self.b: + self.b[self.o : self.o + len(a)] = a + self.o += len(a) + + # TODO: At the moment many of these arguments are named the same as the relevant field + # in the spec, as this is easier to understand. Can save some code size by collapsing them + # down. + + def interface( + self, + bInterfaceNumber, + bNumEndpoints, + bInterfaceClass=_INTERFACE_CLASS_VENDOR, + bInterfaceSubClass=_INTERFACE_SUBCLASS_NONE, + bInterfaceProtocol=_PROTOCOL_NONE, + iInterface=0, + ): + # Utility function to append a standard Interface descriptor, with + # the properties specified in the parameter list. + # + # Defaults for bInterfaceClass, SubClass and Protocol are a "vendor" + # device. + # + # Note that iInterface is a string index number. If set, it should be set + # by the caller Interface to the result of self._get_str_index(s), + # where 's' is a string found in self.strs. + self.pack( + "BBBBBBBBB", + _STD_DESC_INTERFACE_LEN, # bLength + _STD_DESC_INTERFACE_TYPE, # bDescriptorType + bInterfaceNumber, + 0, # bAlternateSetting, not currently supported + bNumEndpoints, + bInterfaceClass, + bInterfaceSubClass, + bInterfaceProtocol, + iInterface, + ) + + def endpoint(self, bEndpointAddress, bmAttributes, wMaxPacketSize, bInterval=1): + # Utility function to append a standard Endpoint descriptor, with + # the properties specified in the parameter list. + # + # See USB 2.0 specification section 9.6.6 Endpoint p269 + # + # As well as a numeric value, bmAttributes can be a string value to represent + # common endpoint types: "control", "bulk", "interrupt". + if bmAttributes == "control": + bmAttributes = 0 + elif bmAttributes == "bulk": + bmAttributes = 2 + elif bmAttributes == "interrupt": + bmAttributes = 3 + + self.pack( + "> 5) & 0x03, + (bmRequestType >> 7) & 0x01, + ) + + +class Buffer: + # An interrupt-safe producer/consumer buffer that wraps a bytearray object. + # + # Kind of like a ring buffer, but supports the idea of returning a + # memoryview for either read or write of multiple bytes (suitable for + # passing to a buffer function without needing to allocate another buffer to + # read into.) + # + # Consumer can call pend_read() to get a memoryview to read from, and then + # finish_read(n) when done to indicate it read 'n' bytes from the + # memoryview. There is also a readinto() convenience function. + # + # Producer must call pend_write() to get a memorybuffer to write into, and + # then finish_write(n) when done to indicate it wrote 'n' bytes into the + # memoryview. There is also a normal write() convenience function. + # + # - Only one producer and one consumer is supported. + # + # - Calling pend_read() and pend_write() is effectively idempotent, they can be + # called more than once without a corresponding finish_x() call if necessary + # (provided only one thread does this, as per the previous point.) + # + # - Calling finish_write() and finish_read() is hard interrupt safe (does + # not allocate). pend_read() and pend_write() each allocate 1 block for + # the memoryview that is returned. + # + # The buffer contents are always laid out as: + # + # - Slice [:_n] = bytes of valid data waiting to read + # - Slice [_n:_w] = unused space + # - Slice [_w:] = bytes of pending write buffer waiting to be written + # + # This buffer should be fast when most reads and writes are balanced and use + # the whole buffer. When this doesn't happen, performance degrades to + # approximate a Python-based single byte ringbuffer. + # + def __init__(self, length): + self._b = memoryview(bytearray(length)) + # number of bytes in buffer read to read, starting at index 0. Updated + # by both producer & consumer. + self._n = 0 + # start index of a pending write into the buffer, if any. equals + # len(self._b) if no write is pending. Updated by producer only. + self._w = length + + def writable(self): + # Number of writable bytes in the buffer. Assumes no pending write is outstanding. + return len(self._b) - self._n + + def readable(self): + # Number of readable bytes in the buffer. Assumes no pending read is outstanding. + return self._n + + def pend_write(self, wmax=None): + # Returns a memoryview that the producer can write bytes into. + # start the write at self._n, the end of data waiting to read + # + # If wmax is set then the memoryview is pre-sliced to be at most + # this many bytes long. + # + # (No critical section needed as self._w is only updated by the producer.) + self._w = self._n + end = (self._w + wmax) if wmax else len(self._b) + return self._b[self._w : end] + + def finish_write(self, nbytes): + # Called by the producer to indicate it wrote nbytes into the buffer. + ist = machine.disable_irq() + try: + assert nbytes <= len(self._b) - self._w # can't say we wrote more than was pended + if self._n == self._w: + # no data was read while the write was happening, so the buffer is already in place + # (this is the fast path) + self._n += nbytes + else: + # Slow path: data was read while the write was happening, so + # shuffle the newly written bytes back towards index 0 to avoid fragmentation + # + # As this updates self._n we have to do it in the critical + # section, so do it byte by byte to avoid allocating. + while nbytes > 0: + self._b[self._n] = self._b[self._w] + self._n += 1 + self._w += 1 + nbytes -= 1 + + self._w = len(self._b) + finally: + machine.enable_irq(ist) + + def write(self, w): + # Helper method for the producer to write into the buffer in one call + pw = self.pend_write() + to_w = min(len(w), len(pw)) + if to_w: + pw[:to_w] = w[:to_w] + self.finish_write(to_w) + return to_w + + def pend_read(self): + # Return a memoryview slice that the consumer can read bytes from + return self._b[: self._n] + + def finish_read(self, nbytes): + # Called by the consumer to indicate it read nbytes from the buffer. + if not nbytes: + return + ist = machine.disable_irq() + try: + assert nbytes <= self._n # can't say we read more than was available + i = 0 + self._n -= nbytes + while i < self._n: + # consumer only read part of the buffer, so shuffle remaining + # read data back towards index 0 to avoid fragmentation + self._b[i] = self._b[i + nbytes] + i += 1 + finally: + machine.enable_irq(ist) + + def readinto(self, b): + # Helper method for the consumer to read out of the buffer in one call + pr = self.pend_read() + to_r = min(len(pr), len(b)) + if to_r: + b[:to_r] = pr[:to_r] + self.finish_read(to_r) + return to_r diff --git a/micropython/utarfile/example-extract.py b/micropython/utarfile/example-extract.py deleted file mode 100644 index a8f828cc9..000000000 --- a/micropython/utarfile/example-extract.py +++ /dev/null @@ -1,13 +0,0 @@ -import sys -import os -import shutil -import utarfile - -t = utarfile.TarFile(sys.argv[1]) -for i in t: - print(i) - if i.type == utarfile.DIRTYPE: - os.makedirs(i.name) - else: - f = t.extractfile(i) - shutil.copyfileobj(f, open(i.name, "wb")) diff --git a/micropython/utarfile/manifest.py b/micropython/utarfile/manifest.py deleted file mode 100644 index 65bd68b9a..000000000 --- a/micropython/utarfile/manifest.py +++ /dev/null @@ -1,5 +0,0 @@ -metadata(description="Lightweight tarfile module subset", version="0.3.2") - -# Originally written by Paul Sokolovsky. - -module("utarfile.py") diff --git a/micropython/utarfile/utarfile.py b/micropython/utarfile/utarfile.py deleted file mode 100644 index 21b899f02..000000000 --- a/micropython/utarfile/utarfile.py +++ /dev/null @@ -1,95 +0,0 @@ -import uctypes - -# http://www.gnu.org/software/tar/manual/html_node/Standard.html -TAR_HEADER = { - "name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100), - "size": (uctypes.ARRAY | 124, uctypes.UINT8 | 11), -} - -DIRTYPE = "dir" -REGTYPE = "file" - - -def roundup(val, align): - return (val + align - 1) & ~(align - 1) - - -class FileSection: - def __init__(self, f, content_len, aligned_len): - self.f = f - self.content_len = content_len - self.align = aligned_len - content_len - - def read(self, sz=65536): - if self.content_len == 0: - return b"" - if sz > self.content_len: - sz = self.content_len - data = self.f.read(sz) - sz = len(data) - self.content_len -= sz - return data - - def readinto(self, buf): - if self.content_len == 0: - return 0 - if len(buf) > self.content_len: - buf = memoryview(buf)[: self.content_len] - sz = self.f.readinto(buf) - self.content_len -= sz - return sz - - def skip(self): - sz = self.content_len + self.align - if sz: - buf = bytearray(16) - while sz: - s = min(sz, 16) - self.f.readinto(buf, s) - sz -= s - - -class TarInfo: - def __str__(self): - return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size) - - -class TarFile: - def __init__(self, name=None, fileobj=None): - if fileobj: - self.f = fileobj - else: - self.f = open(name, "rb") - self.subf = None - - def next(self): - if self.subf: - self.subf.skip() - buf = self.f.read(512) - if not buf: - return None - - h = uctypes.struct(uctypes.addressof(buf), TAR_HEADER, uctypes.LITTLE_ENDIAN) - - # Empty block means end of archive - if h.name[0] == 0: - return None - - d = TarInfo() - d.name = str(h.name, "utf-8").rstrip("\0") - d.size = int(bytes(h.size), 8) - d.type = [REGTYPE, DIRTYPE][d.name[-1] == "/"] - self.subf = d.subf = FileSection(self.f, d.size, roundup(d.size, 512)) - return d - - def __iter__(self): - return self - - def __next__(self): - v = self.next() - if v is None: - raise StopIteration - return v - - def extractfile(self, tarinfo): - return tarinfo.subf diff --git a/micropython/utop/README.md b/micropython/utop/README.md new file mode 100644 index 000000000..7b07e785d --- /dev/null +++ b/micropython/utop/README.md @@ -0,0 +1,12 @@ +# utop + +Provides a top-like live overview of the running system. + +On the `esp32` port this depends on the `esp32.idf_task_info()` function, which +can be enabled by adding the following lines to the board `sdkconfig`: + +```ini +CONFIG_FREERTOS_USE_TRACE_FACILITY=y +CONFIG_FREERTOS_VTASKLIST_INCLUDE_COREID=y +CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS=y +``` diff --git a/micropython/utop/manifest.py b/micropython/utop/manifest.py new file mode 100644 index 000000000..ebba07270 --- /dev/null +++ b/micropython/utop/manifest.py @@ -0,0 +1,6 @@ +metadata( + version="0.1.0", + description="Provides a top-like live overview of the running system.", +) + +module("utop.py") diff --git a/micropython/utop/utop.py b/micropython/utop/utop.py new file mode 100644 index 000000000..799ff4212 --- /dev/null +++ b/micropython/utop/utop.py @@ -0,0 +1,109 @@ +import micropython +import time + +try: + import esp32 + import _thread +except ImportError: + esp32 = None + + +def top(update_interval_ms=1000, timeout_ms=None, thread_names={}): + time_start = time.ticks_ms() + previous_total_runtime = None + previous_task_runtimes = {} + previous_line_count = 0 + esp32_task_state_names = dict( + enumerate(("running", "ready", "blocked", "suspended", "deleted", "invalid")) + ) + + while timeout_ms is None or abs(time.ticks_diff(time.ticks_ms(), time_start)) < timeout_ms: + if previous_line_count > 0: + print("\x1b[{}A".format(previous_line_count), end="") + line_count = 0 + + if esp32 is not None: + if not hasattr(esp32, "idf_task_info"): + print( + "INFO: esp32.idf_task_info() is not available, cannot list active tasks.\x1b[K" + ) + line_count += 1 + else: + print(" CPU% CORE PRIORITY STATE STACKWATERMARK NAME\x1b[K") + line_count += 1 + + total_runtime, tasks = esp32.idf_task_info() + current_thread_id = _thread.get_ident() + tasks.sort(key=lambda t: t[0]) + for ( + task_id, + task_name, + task_state, + task_priority, + task_runtime, + task_stackhighwatermark, + task_coreid, + ) in tasks: + task_runtime_percentage = "-" + if ( + total_runtime != previous_total_runtime + and task_id in previous_task_runtimes + ): + task_runtime_percentage = "{:.2f}%".format( + 100 + * ((task_runtime - previous_task_runtimes[task_id]) & (2**32 - 1)) + / ((total_runtime - previous_total_runtime) & (2**32 - 1)) + ) + print( + "{:>7} {:>4} {:>8d} {:<9} {:<14d} {}{}\x1b[K".format( + task_runtime_percentage, + "-" if task_coreid is None else task_coreid, + task_priority, + esp32_task_state_names.get(task_state, "unknown"), + task_stackhighwatermark, + thread_names.get(task_id, task_name), + " (*)" if task_id == current_thread_id else "", + ) + ) + line_count += 1 + + previous_task_runtimes[task_id] = task_runtime + previous_total_runtime = total_runtime + else: + print("INFO: Platform does not support listing active tasks.\x1b[K") + line_count += 1 + + print("\x1b[K") + line_count += 1 + print("MicroPython ", end="") + micropython.mem_info() + line_count += 3 + + if esp32 is not None: + print("\x1b[K") + line_count += 1 + for name, cap in (("data", esp32.HEAP_DATA), ("exec", esp32.HEAP_EXEC)): + heaps = esp32.idf_heap_info(cap) + print( + "IDF heap ({}): {} regions, {} total, {} free, {} largest contiguous, {} min free watermark\x1b[K".format( + name, + len(heaps), + sum((h[0] for h in heaps)), + sum((h[1] for h in heaps)), + max((h[2] for h in heaps)), + sum((h[3] for h in heaps)), + ) + ) + line_count += 1 + + if previous_line_count > line_count: + for _ in range(previous_line_count - line_count): + print("\x1b[K") + print("\x1b[{}A".format(previous_line_count - line_count), end="") + + previous_line_count = line_count + + try: + time.sleep_ms(update_interval_ms) + except KeyboardInterrupt: + break diff --git a/micropython/xmltok/manifest.py b/micropython/xmltok/manifest.py index efbe41cc4..70d5556bf 100644 --- a/micropython/xmltok/manifest.py +++ b/micropython/xmltok/manifest.py @@ -1,4 +1,4 @@ -metadata(description="Simple XML tokenizer", version="0.2") +metadata(description="Simple XML tokenizer", version="0.2.1") # Originally written by Paul Sokolovsky. diff --git a/micropython/xmltok/xmltok.py b/micropython/xmltok/xmltok.py index 53435d073..9711b7ff2 100644 --- a/micropython/xmltok/xmltok.py +++ b/micropython/xmltok/xmltok.py @@ -31,7 +31,7 @@ def eof(self): def nextch(self): self.c = self.f.read(1) if not self.c: - raise StopIteration + raise EOFError return self.c def skip_ws(self): @@ -87,36 +87,39 @@ def lex_attrs_till(self): def tokenize(self): while not self.eof(): - if self.match("<"): - if self.match("/"): - yield (END_TAG, self.getnsident()) - self.expect(">") - elif self.match("?"): - yield (PI, self.getident()) - yield from self.lex_attrs_till() - self.expect("?") - self.expect(">") - elif self.match("!"): - self.expect("-") - self.expect("-") - last3 = "" - while True: - last3 = last3[-2:] + self.getch() - if last3 == "-->": - break - else: - tag = self.getnsident() - yield (START_TAG, tag) - yield from self.lex_attrs_till() + try: + if self.match("<"): if self.match("/"): - yield (END_TAG, tag) - self.expect(">") - else: - text = "" - while self.curch() != "<": - text += self.getch() - if text: - yield (TEXT, text) + yield (END_TAG, self.getnsident()) + self.expect(">") + elif self.match("?"): + yield (PI, self.getident()) + yield from self.lex_attrs_till() + self.expect("?") + self.expect(">") + elif self.match("!"): + self.expect("-") + self.expect("-") + last3 = "" + while True: + last3 = last3[-2:] + self.getch() + if last3 == "-->": + break + else: + tag = self.getnsident() + yield (START_TAG, tag) + yield from self.lex_attrs_till() + if self.match("/"): + yield (END_TAG, tag) + self.expect(">") + else: + text = "" + while self.curch() != "<": + text += self.getch() + if text: + yield (TEXT, text) + except EOFError: + pass def gfind(gen, pred): diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..83d29405d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,101 @@ +[tool.ruff] +exclude = [ + "python-stdlib", + "unix-ffi", +] +select = [ + "ASYNC", # flake8-comprehensions + "C4", # flake8-comprehensions + "C90", # McCabe cyclomatic complexity + "DTZ", # flake8-datetimez + "E", # pycodestyle + "EXE", # flake8-executable + "F", # Pyflakes + "G", # flake8-logging-format + "ICN", # flake8-import-conventions + "INT", # flake8-gettext + "ISC", # flake8-implicit-str-concat + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # Pylint + "PYI", # flake8-pyi + "RSE", # flake8-raise + "RUF", # Ruff-specific rules + "T10", # flake8-debugger + "TCH", # flake8-type-checking + "W", # pycodestyle + "YTT", # flake8-2020 + # "A", # flake8-builtins + # "ANN", # flake8-annotations + # "ARG", # flake8-unused-arguments + # "B", # flake8-bugbear + # "BLE", # flake8-blind-except + # "COM", # flake8-commas + # "D", # pydocstyle + # "DJ", # flake8-django + # "EM", # flake8-errmsg + # "ERA", # eradicate + # "FBT", # flake8-boolean-trap + # "I", # isort + # "INP", # flake8-no-pep420 + # "N", # pep8-naming + # "NPY", # NumPy-specific rules + # "PD", # pandas-vet + # "PT", # flake8-pytest-style + # "PTH", # flake8-use-pathlib + # "Q", # flake8-quotes + # "RET", # flake8-return + # "S", # flake8-bandit + # "SIM", # flake8-simplify + # "SLF", # flake8-self + # "T20", # flake8-print + # "TID", # flake8-tidy-imports + # "TRY", # tryceratops + # "UP", # pyupgrade +] +ignore = [ + "E722", + "E741", # 'l' is currently widely used + "F401", + "F403", + "F405", + "E501", # line length, recommended to disable + "ISC001", + "ISC003", # micropython does not support implicit concatenation of f-strings + "PIE810", # micropython does not support passing tuples to .startswith or .endswith + "PLC1901", + "PLR1704", # sometimes desirable to redefine an argument to save code size + "PLR1714", + "PLR5501", + "PLW0602", + "PLW0603", + "PLW2901", + "RUF012", + "RUF100", + "SIM101", + "W191", # tab-indent, redundant when using formatter +] +line-length = 99 +target-version = "py37" + +[tool.ruff.mccabe] +max-complexity = 61 + +[tool.ruff.pylint] +allow-magic-value-types = ["bytes", "int", "str"] +max-args = 14 +max-branches = 58 +max-returns = 13 +max-statements = 166 + +[tool.ruff.per-file-ignores] +"micropython/aiorepl/aiorepl.py" = ["PGH001"] + +# manifest.py files are evaluated with some global names pre-defined +"**/manifest.py" = ["F821"] +"ports/**/boards/manifest*.py" = ["F821"] + +# ble multitests are evaluated with some names pre-defined +"micropython/bluetooth/aioble/multitests/*" = ["F821"] + +[tool.ruff.format] diff --git a/python-ecosys/aiohttp/README.md b/python-ecosys/aiohttp/README.md new file mode 100644 index 000000000..5ce5e14bc --- /dev/null +++ b/python-ecosys/aiohttp/README.md @@ -0,0 +1,32 @@ +aiohttp is an HTTP client module for MicroPython asyncio module, +with API mostly compatible with CPython [aiohttp](https://github.com/aio-libs/aiohttp) +module. + +> [!NOTE] +> Only client is implemented. + +See `examples/client.py` +```py +import aiohttp +import asyncio + +async def main(): + + async with aiohttp.ClientSession() as session: + async with session.get('http://micropython.org/') as response: + + print("Status:", response.status) + print("Content-Type:", response.headers['Content-Type']) + + html = await response.text() + print("Body:", html[:15], "...") + +asyncio.run(main()) +``` +``` +$ micropython examples/client.py +Status: 200 +Content-Type: text/html; charset=utf-8 +Body: ... + +``` diff --git a/python-ecosys/aiohttp/aiohttp/__init__.py b/python-ecosys/aiohttp/aiohttp/__init__.py new file mode 100644 index 000000000..8c5493f30 --- /dev/null +++ b/python-ecosys/aiohttp/aiohttp/__init__.py @@ -0,0 +1,275 @@ +# MicroPython aiohttp library +# MIT license; Copyright (c) 2023 Carlos Gil + +import asyncio +import json as _json +from .aiohttp_ws import ( + _WSRequestContextManager, + ClientWebSocketResponse, + WebSocketClient, + WSMsgType, +) + +HttpVersion10 = "HTTP/1.0" +HttpVersion11 = "HTTP/1.1" + + +class ClientResponse: + def __init__(self, reader): + self.content = reader + + def _get_header(self, keyname, default): + for k in self.headers: + if k.lower() == keyname: + return self.headers[k] + return default + + def _decode(self, data): + c_encoding = self._get_header("content-encoding", None) + if c_encoding in ("gzip", "deflate", "gzip,deflate"): + try: + import deflate + import io + + if c_encoding == "deflate": + with deflate.DeflateIO(io.BytesIO(data), deflate.ZLIB) as d: + return d.read() + elif c_encoding == "gzip": + with deflate.DeflateIO(io.BytesIO(data), deflate.GZIP, 15) as d: + return d.read() + except ImportError: + print("WARNING: deflate module required") + return data + + async def read(self, sz=-1): + return self._decode(await self.content.read(sz)) + + async def text(self, encoding="utf-8"): + return (await self.read(int(self._get_header("content-length", -1)))).decode(encoding) + + async def json(self): + return _json.loads(await self.read(int(self._get_header("content-length", -1)))) + + def __repr__(self): + return "" % (self.status, self.headers) + + +class ChunkedClientResponse(ClientResponse): + def __init__(self, reader): + self.content = reader + self.chunk_size = 0 + + async def read(self, sz=4 * 1024 * 1024): + if self.chunk_size == 0: + l = await self.content.readline() + l = l.split(b";", 1)[0] + self.chunk_size = int(l, 16) + if self.chunk_size == 0: + # End of message + sep = await self.content.read(2) + assert sep == b"\r\n" + return b"" + data = await self.content.read(min(sz, self.chunk_size)) + self.chunk_size -= len(data) + if self.chunk_size == 0: + sep = await self.content.read(2) + assert sep == b"\r\n" + return self._decode(data) + + def __repr__(self): + return "" % (self.status, self.headers) + + +class _RequestContextManager: + def __init__(self, client, request_co): + self.reqco = request_co + self.client = client + + async def __aenter__(self): + return await self.reqco + + async def __aexit__(self, *args): + await self.client._reader.aclose() + return await asyncio.sleep(0) + + +class ClientSession: + def __init__(self, base_url="", headers={}, version=HttpVersion10): + self._reader = None + self._base_url = base_url + self._base_headers = {"Connection": "close", "User-Agent": "compat"} + self._base_headers.update(**headers) + self._http_version = version + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return await asyncio.sleep(0) + + # TODO: Implement timeouts + + async def _request(self, method, url, data=None, json=None, ssl=None, params=None, headers={}): + redir_cnt = 0 + while redir_cnt < 2: + reader = await self.request_raw(method, url, data, json, ssl, params, headers) + _headers = [] + sline = await reader.readline() + sline = sline.split(None, 2) + status = int(sline[1]) + chunked = False + while True: + line = await reader.readline() + if not line or line == b"\r\n": + break + _headers.append(line) + if line.startswith(b"Transfer-Encoding:"): + if b"chunked" in line: + chunked = True + elif line.startswith(b"Location:"): + url = line.rstrip().split(None, 1)[1].decode() + + if 301 <= status <= 303: + redir_cnt += 1 + await reader.aclose() + continue + break + + if chunked: + resp = ChunkedClientResponse(reader) + else: + resp = ClientResponse(reader) + resp.status = status + resp.headers = _headers + resp.url = url + if params: + resp.url += "?" + "&".join(f"{k}={params[k]}" for k in sorted(params)) + try: + resp.headers = { + val.split(":", 1)[0]: val.split(":", 1)[-1].strip() + for val in [hed.decode().strip() for hed in _headers] + } + except Exception: + pass + self._reader = reader + return resp + + async def request_raw( + self, + method, + url, + data=None, + json=None, + ssl=None, + params=None, + headers={}, + is_handshake=False, + version=None, + ): + if json and isinstance(json, dict): + data = _json.dumps(json) + if data is not None and method == "GET": + method = "POST" + if params: + url += "?" + "&".join(f"{k}={params[k]}" for k in sorted(params)) + try: + proto, dummy, host, path = url.split("/", 3) + except ValueError: + proto, dummy, host = url.split("/", 2) + path = "" + + if proto == "http:": + port = 80 + elif proto == "https:": + port = 443 + if ssl is None: + ssl = True + else: + raise ValueError("Unsupported protocol: " + proto) + + if ":" in host: + host, port = host.split(":", 1) + port = int(port) + + reader, writer = await asyncio.open_connection(host, port, ssl=ssl) + + # Use protocol 1.0, because 1.1 always allows to use chunked transfer-encoding + # But explicitly set Connection: close, even though this should be default for 1.0, + # because some servers misbehave w/o it. + if version is None: + version = self._http_version + if "Host" not in headers: + headers.update(Host=host) + if not data: + query = b"%s /%s %s\r\n%s\r\n" % ( + method, + path, + version, + "\r\n".join(f"{k}: {v}" for k, v in headers.items()) + "\r\n" if headers else "", + ) + else: + if json: + headers.update(**{"Content-Type": "application/json"}) + if isinstance(data, bytes): + headers.update(**{"Content-Type": "application/octet-stream"}) + else: + data = data.encode() + + headers.update(**{"Content-Length": len(data)}) + query = b"""%s /%s %s\r\n%s\r\n%s""" % ( + method, + path, + version, + "\r\n".join(f"{k}: {v}" for k, v in headers.items()) + "\r\n", + data, + ) + if not is_handshake: + await writer.awrite(query) + return reader + else: + await writer.awrite(query) + return reader, writer + + def request(self, method, url, data=None, json=None, ssl=None, params=None, headers={}): + return _RequestContextManager( + self, + self._request( + method, + self._base_url + url, + data=data, + json=json, + ssl=ssl, + params=params, + headers=dict(**self._base_headers, **headers), + ), + ) + + def get(self, url, **kwargs): + return self.request("GET", url, **kwargs) + + def post(self, url, **kwargs): + return self.request("POST", url, **kwargs) + + def put(self, url, **kwargs): + return self.request("PUT", url, **kwargs) + + def patch(self, url, **kwargs): + return self.request("PATCH", url, **kwargs) + + def delete(self, url, **kwargs): + return self.request("DELETE", url, **kwargs) + + def head(self, url, **kwargs): + return self.request("HEAD", url, **kwargs) + + def options(self, url, **kwargs): + return self.request("OPTIONS", url, **kwargs) + + def ws_connect(self, url, ssl=None): + return _WSRequestContextManager(self, self._ws_connect(url, ssl=ssl)) + + async def _ws_connect(self, url, ssl=None): + ws_client = WebSocketClient(self._base_headers.copy()) + await ws_client.connect(url, ssl=ssl, handshake_request=self.request_raw) + self._reader = ws_client.reader + return ClientWebSocketResponse(ws_client) diff --git a/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py b/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py new file mode 100644 index 000000000..6e0818c92 --- /dev/null +++ b/python-ecosys/aiohttp/aiohttp/aiohttp_ws.py @@ -0,0 +1,269 @@ +# MicroPython aiohttp library +# MIT license; Copyright (c) 2023 Carlos Gil +# adapted from https://github.com/danni/uwebsockets +# and https://github.com/miguelgrinberg/microdot/blob/main/src/microdot_asyncio_websocket.py + +import asyncio +import random +import json as _json +import binascii +import re +import struct +from collections import namedtuple + +URL_RE = re.compile(r"(wss|ws)://([A-Za-z0-9-\.]+)(?:\:([0-9]+))?(/.+)?") +URI = namedtuple("URI", ("protocol", "hostname", "port", "path")) # noqa: PYI024 + + +def urlparse(uri): + """Parse ws:// URLs""" + match = URL_RE.match(uri) + if match: + protocol = match.group(1) + host = match.group(2) + port = match.group(3) + path = match.group(4) + + if protocol == "wss": + if port is None: + port = 443 + elif protocol == "ws": + if port is None: + port = 80 + else: + raise ValueError("Scheme {} is invalid".format(protocol)) + + return URI(protocol, host, int(port), path) + + +class WebSocketMessage: + def __init__(self, opcode, data): + self.type = opcode + self.data = data + + +class WSMsgType: + TEXT = 1 + BINARY = 2 + ERROR = 258 + + +class WebSocketClient: + CONT = 0 + TEXT = 1 + BINARY = 2 + CLOSE = 8 + PING = 9 + PONG = 10 + + def __init__(self, params): + self.params = params + self.closed = False + self.reader = None + self.writer = None + + async def connect(self, uri, ssl=None, handshake_request=None): + uri = urlparse(uri) + assert uri + if uri.protocol == "wss": + if not ssl: + ssl = True + await self.handshake(uri, ssl, handshake_request) + + @classmethod + def _parse_frame_header(cls, header): + byte1, byte2 = struct.unpack("!BB", header) + + # Byte 1: FIN(1) _(1) _(1) _(1) OPCODE(4) + fin = bool(byte1 & 0x80) + opcode = byte1 & 0x0F + + # Byte 2: MASK(1) LENGTH(7) + mask = bool(byte2 & (1 << 7)) + length = byte2 & 0x7F + + return fin, opcode, mask, length + + def _process_websocket_frame(self, opcode, payload): + if opcode == self.TEXT: + payload = str(payload, "utf-8") + elif opcode == self.BINARY: + pass + elif opcode == self.CLOSE: + # raise OSError(32, "Websocket connection closed") + return opcode, payload + elif opcode == self.PING: + return self.PONG, payload + elif opcode == self.PONG: # pragma: no branch + return None, None + return None, payload + + @classmethod + def _encode_websocket_frame(cls, opcode, payload): + if opcode == cls.TEXT: + payload = payload.encode() + + length = len(payload) + fin = mask = True + + # Frame header + # Byte 1: FIN(1) _(1) _(1) _(1) OPCODE(4) + byte1 = 0x80 if fin else 0 + byte1 |= opcode + + # Byte 2: MASK(1) LENGTH(7) + byte2 = 0x80 if mask else 0 + + if length < 126: # 126 is magic value to use 2-byte length header + byte2 |= length + frame = struct.pack("!BB", byte1, byte2) + + elif length < (1 << 16): # Length fits in 2-bytes + byte2 |= 126 # Magic code + frame = struct.pack("!BBH", byte1, byte2, length) + + elif length < (1 << 64): + byte2 |= 127 # Magic code + frame = struct.pack("!BBQ", byte1, byte2, length) + + else: + raise ValueError + + # Mask is 4 bytes + mask_bits = struct.pack("!I", random.getrandbits(32)) + frame += mask_bits + payload = bytes(b ^ mask_bits[i % 4] for i, b in enumerate(payload)) + return frame + payload + + async def handshake(self, uri, ssl, req): + headers = self.params + _http_proto = "http" if uri.protocol != "wss" else "https" + url = f"{_http_proto}://{uri.hostname}:{uri.port}{uri.path or '/'}" + key = binascii.b2a_base64(bytes(random.getrandbits(8) for _ in range(16)))[:-1] + headers["Host"] = f"{uri.hostname}:{uri.port}" + headers["Connection"] = "Upgrade" + headers["Upgrade"] = "websocket" + headers["Sec-WebSocket-Key"] = str(key, "utf-8") + headers["Sec-WebSocket-Version"] = "13" + headers["Origin"] = f"{_http_proto}://{uri.hostname}:{uri.port}" + + self.reader, self.writer = await req( + "GET", + url, + ssl=ssl, + headers=headers, + is_handshake=True, + version="HTTP/1.1", + ) + + header = await self.reader.readline() + header = header[:-2] + assert header.startswith(b"HTTP/1.1 101 "), header + + while header: + header = await self.reader.readline() + header = header[:-2] + + async def receive(self): + while True: + opcode, payload = await self._read_frame() + send_opcode, data = self._process_websocket_frame(opcode, payload) + if send_opcode: # pragma: no cover + await self.send(data, send_opcode) + if opcode == self.CLOSE: + self.closed = True + return opcode, data + elif data: # pragma: no branch + return opcode, data + + async def send(self, data, opcode=None): + frame = self._encode_websocket_frame( + opcode or (self.TEXT if isinstance(data, str) else self.BINARY), data + ) + self.writer.write(frame) + await self.writer.drain() + + async def close(self): + if not self.closed: # pragma: no cover + self.closed = True + await self.send(b"", self.CLOSE) + + async def _read_frame(self): + header = await self.reader.read(2) + if len(header) != 2: # pragma: no cover + # raise OSError(32, "Websocket connection closed") + opcode = self.CLOSE + payload = b"" + return opcode, payload + fin, opcode, has_mask, length = self._parse_frame_header(header) + if length == 126: # Magic number, length header is 2 bytes + (length,) = struct.unpack("!H", await self.reader.read(2)) + elif length == 127: # Magic number, length header is 8 bytes + (length,) = struct.unpack("!Q", await self.reader.read(8)) + + if has_mask: # pragma: no cover + mask = await self.reader.read(4) + payload = await self.reader.read(length) + if has_mask: # pragma: no cover + payload = bytes(x ^ mask[i % 4] for i, x in enumerate(payload)) + return opcode, payload + + +class ClientWebSocketResponse: + def __init__(self, wsclient): + self.ws = wsclient + + def __aiter__(self): + return self + + async def __anext__(self): + msg = WebSocketMessage(*await self.ws.receive()) + # print(msg.data, msg.type) # DEBUG + if (not msg.data and msg.type == self.ws.CLOSE) or self.ws.closed: + raise StopAsyncIteration + return msg + + async def close(self): + await self.ws.close() + + async def send_str(self, data): + if not isinstance(data, str): + raise TypeError("data argument must be str (%r)" % type(data)) + await self.ws.send(data) + + async def send_bytes(self, data): + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError("data argument must be byte-ish (%r)" % type(data)) + await self.ws.send(data) + + async def send_json(self, data): + await self.send_str(_json.dumps(data)) + + async def receive_str(self): + msg = WebSocketMessage(*await self.ws.receive()) + if msg.type != self.ws.TEXT: + raise TypeError(f"Received message {msg.type}:{msg.data!r} is not str") + return msg.data + + async def receive_bytes(self): + msg = WebSocketMessage(*await self.ws.receive()) + if msg.type != self.ws.BINARY: + raise TypeError(f"Received message {msg.type}:{msg.data!r} is not bytes") + return msg.data + + async def receive_json(self): + data = await self.receive_str() + return _json.loads(data) + + +class _WSRequestContextManager: + def __init__(self, client, request_co): + self.reqco = request_co + self.client = client + + async def __aenter__(self): + return await self.reqco + + async def __aexit__(self, *args): + await self.client._reader.aclose() + return await asyncio.sleep(0) diff --git a/python-ecosys/aiohttp/examples/client.py b/python-ecosys/aiohttp/examples/client.py new file mode 100644 index 000000000..0a6476064 --- /dev/null +++ b/python-ecosys/aiohttp/examples/client.py @@ -0,0 +1,19 @@ +import sys + +# ruff: noqa: E402 +sys.path.insert(0, ".") +import aiohttp +import asyncio + + +async def main(): + async with aiohttp.ClientSession() as session: + async with session.get("http://micropython.org/") as response: + print("Status:", response.status) + print("Content-Type:", response.headers["Content-Type"]) + + html = await response.text() + print("Body:", html[:15], "...") + + +asyncio.run(main()) diff --git a/python-ecosys/aiohttp/examples/compression.py b/python-ecosys/aiohttp/examples/compression.py new file mode 100644 index 000000000..a1c6276b2 --- /dev/null +++ b/python-ecosys/aiohttp/examples/compression.py @@ -0,0 +1,21 @@ +import sys + +# ruff: noqa: E402 +sys.path.insert(0, ".") +import aiohttp +import asyncio + +headers = {"Accept-Encoding": "gzip,deflate"} + + +async def main(): + async with aiohttp.ClientSession(headers=headers, version=aiohttp.HttpVersion11) as session: + async with session.get("http://micropython.org/") as response: + print("Status:", response.status) + print("Content-Type:", response.headers["Content-Type"]) + print(response.headers) + html = await response.text() + print(html) + + +asyncio.run(main()) diff --git a/python-ecosys/aiohttp/examples/get.py b/python-ecosys/aiohttp/examples/get.py new file mode 100644 index 000000000..087d6fb51 --- /dev/null +++ b/python-ecosys/aiohttp/examples/get.py @@ -0,0 +1,30 @@ +import sys + +# ruff: noqa: E402 +sys.path.insert(0, ".") +import aiohttp +import asyncio + + +URL = sys.argv.pop() + +if not URL.startswith("http"): + URL = "http://micropython.org/" + +print(URL) + + +async def fetch(client): + async with client.get(URL) as resp: + assert resp.status == 200 + return await resp.text() + + +async def main(): + async with aiohttp.ClientSession() as client: + html = await fetch(client) + print(html) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python-ecosys/aiohttp/examples/headers.py b/python-ecosys/aiohttp/examples/headers.py new file mode 100644 index 000000000..ec5c00a80 --- /dev/null +++ b/python-ecosys/aiohttp/examples/headers.py @@ -0,0 +1,19 @@ +import sys + +# ruff: noqa: E402 +sys.path.insert(0, ".") +import aiohttp +import asyncio + + +headers = {"Authorization": "Basic bG9naW46cGFzcw=="} + + +async def main(): + async with aiohttp.ClientSession(headers=headers) as session: + async with session.get("http://httpbin.org/headers") as r: + json_body = await r.json() + print(json_body) + + +asyncio.run(main()) diff --git a/python-ecosys/aiohttp/examples/methods.py b/python-ecosys/aiohttp/examples/methods.py new file mode 100644 index 000000000..af38ff652 --- /dev/null +++ b/python-ecosys/aiohttp/examples/methods.py @@ -0,0 +1,26 @@ +import sys + +# ruff: noqa: E402 +sys.path.insert(0, ".") +import aiohttp +import asyncio + + +async def main(): + async with aiohttp.ClientSession("http://httpbin.org/") as session: + async with session.get("/get") as resp: + assert resp.status == 200 + rget = await resp.text() + print(f"GET: {rget}") + async with session.post("/post", json={"foo": "bar"}) as resp: + assert resp.status == 200 + rpost = await resp.text() + print(f"POST: {rpost}") + async with session.put("/put", data=b"data") as resp: + assert resp.status == 200 + rput = await resp.json() + print("PUT: ", rput) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python-ecosys/aiohttp/examples/params.py b/python-ecosys/aiohttp/examples/params.py new file mode 100644 index 000000000..9aecb2ab8 --- /dev/null +++ b/python-ecosys/aiohttp/examples/params.py @@ -0,0 +1,21 @@ +import sys + +# ruff: noqa: E402 +sys.path.insert(0, ".") +import aiohttp +import asyncio + + +params = {"key1": "value1", "key2": "value2"} + + +async def main(): + async with aiohttp.ClientSession() as session: + async with session.get("http://httpbin.org/get", params=params) as response: + expect = "http://httpbin.org/get?key1=value1&key2=value2" + assert str(response.url) == expect, f"{response.url} != {expect}" + html = await response.text() + print(html) + + +asyncio.run(main()) diff --git a/python-ecosys/aiohttp/examples/ws.py b/python-ecosys/aiohttp/examples/ws.py new file mode 100644 index 000000000..b96ee6819 --- /dev/null +++ b/python-ecosys/aiohttp/examples/ws.py @@ -0,0 +1,45 @@ +import sys + +# ruff: noqa: E402 +sys.path.insert(0, ".") +import aiohttp +import asyncio + +try: + URL = sys.argv[1] # expects a websocket echo server +except Exception: + URL = "ws://echo.websocket.events" + + +sslctx = False + +if URL.startswith("wss:"): + try: + import ssl + + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslctx.verify_mode = ssl.CERT_NONE + except Exception: + pass + + +async def ws_test_echo(session): + async with session.ws_connect(URL, ssl=sslctx) as ws: + await ws.send_str("hello world!\r\n") + async for msg in ws: + if msg.type == aiohttp.WSMsgType.TEXT: + print(msg.data) + + if "close" in msg.data: + break + await ws.send_str("close\r\n") + await ws.close() + + +async def main(): + async with aiohttp.ClientSession() as session: + await ws_test_echo(session) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python-ecosys/aiohttp/examples/ws_repl_echo.py b/python-ecosys/aiohttp/examples/ws_repl_echo.py new file mode 100644 index 000000000..c41a4ee5e --- /dev/null +++ b/python-ecosys/aiohttp/examples/ws_repl_echo.py @@ -0,0 +1,54 @@ +import sys + +# ruff: noqa: E402 +sys.path.insert(0, ".") +import aiohttp +import asyncio + +try: + URL = sys.argv[1] # expects a websocket echo server + READ_BANNER = False +except Exception: + URL = "ws://echo.websocket.events" + READ_BANNER = True + + +sslctx = False + +if URL.startswith("wss:"): + try: + import ssl + + sslctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + sslctx.verify_mode = ssl.CERT_NONE + except Exception: + pass + + +async def ws_test_echo(session): + async with session.ws_connect(URL, ssl=sslctx) as ws: + if READ_BANNER: + print(await ws.receive_str()) + try: + while True: + await ws.send_str(f"{input('>>> ')}\r\n") + + async for msg in ws: + if msg.type == aiohttp.WSMsgType.TEXT: + print(msg.data, end="") + break + + except KeyboardInterrupt: + pass + + finally: + await ws.close() + + +async def main(): + async with aiohttp.ClientSession() as session: + await ws_test_echo(session) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/python-ecosys/aiohttp/manifest.py b/python-ecosys/aiohttp/manifest.py new file mode 100644 index 000000000..d22a6ce11 --- /dev/null +++ b/python-ecosys/aiohttp/manifest.py @@ -0,0 +1,7 @@ +metadata( + description="HTTP client module for MicroPython asyncio module", + version="0.0.5", + pypi="aiohttp", +) + +package("aiohttp") diff --git a/python-ecosys/cbor2/cbor2/__init__.py b/python-ecosys/cbor2/cbor2/__init__.py new file mode 100644 index 000000000..80790f0da --- /dev/null +++ b/python-ecosys/cbor2/cbor2/__init__.py @@ -0,0 +1,32 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +from ._decoder import CBORDecoder +from ._decoder import load +from ._decoder import loads + +from ._encoder import CBOREncoder +from ._encoder import dump +from ._encoder import dumps diff --git a/python-ecosys/cbor2/cbor2/_decoder.py b/python-ecosys/cbor2/cbor2/_decoder.py new file mode 100644 index 000000000..965dbfd46 --- /dev/null +++ b/python-ecosys/cbor2/cbor2/_decoder.py @@ -0,0 +1,262 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import io +import struct + + +class CBORDecodeError(Exception): + """Raised when an error occurs deserializing a CBOR datastream.""" + + +break_marker = object() + + +class CBORSimpleValue(object): + """ + Represents a CBOR "simple value". + :param int value: the value (0-255) + """ + + def __init__(self, value): + if value < 0 or value > 255: + raise TypeError("simple value too big") + self.value = value + + def __eq__(self, other): + if isinstance(other, CBORSimpleValue): + return self.value == other.value + elif isinstance(other, int): + return self.value == other + return NotImplemented + + def __repr__(self): + return "CBORSimpleValue({self.value})".format(self=self) + + +def decode_uint(decoder, subtype, allow_indefinite=False): + # Major tag 0 + if subtype < 24: + return subtype + elif subtype == 24: + return struct.unpack(">B", decoder.read(1))[0] + elif subtype == 25: + return struct.unpack(">H", decoder.read(2))[0] + elif subtype == 26: + return struct.unpack(">L", decoder.read(4))[0] + elif subtype == 27: + return struct.unpack(">Q", decoder.read(8))[0] + elif subtype == 31 and allow_indefinite: + return None + else: + raise CBORDecodeError("unknown unsigned integer subtype 0x%x" % subtype) + + +def decode_negint(decoder, subtype): + # Major tag 1 + uint = decode_uint(decoder, subtype) + return -uint - 1 + + +def decode_bytestring(decoder, subtype): + # Major tag 2 + length = decode_uint(decoder, subtype, allow_indefinite=True) + if length is None: + # Indefinite length + buf = bytearray() + while True: + initial_byte = decoder.read(1)[0] + if initial_byte == 255: + return buf + else: + length = decode_uint(decoder, initial_byte & 31) + value = decoder.read(length) + buf.extend(value) + else: + return decoder.read(length) + + +def decode_string(decoder, subtype): + # Major tag 3 + return decode_bytestring(decoder, subtype).decode("utf-8") + + +def decode_array(decoder, subtype): + # Major tag 4 + items = [] + length = decode_uint(decoder, subtype, allow_indefinite=True) + if length is None: + # Indefinite length + while True: + value = decoder.decode() + if value is break_marker: + break + else: + items.append(value) + else: + for _ in range(length): + item = decoder.decode() + items.append(item) + return items + + +def decode_map(decoder, subtype): + # Major tag 5 + dictionary = {} + length = decode_uint(decoder, subtype, allow_indefinite=True) + if length is None: + # Indefinite length + while True: + key = decoder.decode() + if key is break_marker: + break + else: + value = decoder.decode() + dictionary[key] = value + else: + for _ in range(length): + key = decoder.decode() + value = decoder.decode() + dictionary[key] = value + + return dictionary + + +def decode_special(decoder, subtype): + # Simple value + if subtype < 20: + return CBORSimpleValue(subtype) + + # Major tag 7 + return special_decoders[subtype](decoder) + + +def decode_simple_value(decoder): + return CBORSimpleValue(struct.unpack(">B", decoder.read(1))[0]) + + +def decode_float16(decoder): + decoder.read(2) + raise NotImplementedError # no float16 unpack function + + +def decode_float32(decoder): + return struct.unpack(">f", decoder.read(4))[0] + + +def decode_float64(decoder): + return struct.unpack(">d", decoder.read(8))[0] + + +major_decoders = { + 0: decode_uint, + 1: decode_negint, + 2: decode_bytestring, + 3: decode_string, + 4: decode_array, + 5: decode_map, + 7: decode_special, +} + +special_decoders = { + 20: lambda self: False, + 21: lambda self: True, + 22: lambda self: None, + # 23 is undefined + 24: decode_simple_value, + 25: decode_float16, + 26: decode_float32, + 27: decode_float64, + 31: lambda self: break_marker, +} + + +class CBORDecoder(object): + """ + Deserializes a CBOR encoded byte stream. + """ + + def __init__(self, fp): + self.fp = fp + + def read(self, amount): + """ + Read bytes from the data stream. + :param int amount: the number of bytes to read + """ + data = self.fp.read(amount) + if len(data) < amount: + raise CBORDecodeError( + "premature end of stream (expected to read {} bytes, got {} instead)".format( + amount, len(data) + ) + ) + + return data + + def decode(self): + """ + Decode the next value from the stream. + :raises CBORDecodeError: if there is any problem decoding the stream + """ + try: + initial_byte = self.fp.read(1)[0] + major_type = initial_byte >> 5 + subtype = initial_byte & 31 + except Exception as e: + raise CBORDecodeError( + "error reading major type at index {}: {}".format(self.fp.tell(), e) + ) + + decoder = major_decoders[major_type] + try: + return decoder(self, subtype) + except CBORDecodeError: + raise + except Exception as e: + raise CBORDecodeError( + "error decoding value {}".format(e) + ) # tell doesn't work on micropython at the moment + + +def loads(payload, **kwargs): + """ + Deserialize an object from a bytestring. + :param bytes payload: the bytestring to serialize + :param kwargs: keyword arguments passed to :class:`~.CBORDecoder` + :return: the deserialized object + """ + fp = io.BytesIO(payload) + return CBORDecoder(fp, **kwargs).decode() + + +def load(fp, **kwargs): + """ + Deserialize an object from an open file. + :param fp: the input file (any file-like object) + :param kwargs: keyword arguments passed to :class:`~.CBORDecoder` + :return: the deserialized object + """ + return CBORDecoder(fp, **kwargs).decode() diff --git a/python-ecosys/cbor2/cbor2/_encoder.py b/python-ecosys/cbor2/cbor2/_encoder.py new file mode 100644 index 000000000..fe8715468 --- /dev/null +++ b/python-ecosys/cbor2/cbor2/_encoder.py @@ -0,0 +1,182 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import io +import struct + + +class CBOREncodeError(Exception): + """Raised when an error occurs while serializing an object into a CBOR datastream.""" + + +def encode_length(major_tag, length): + if length < 24: + return struct.pack(">B", major_tag | length) + elif length < 256: + return struct.pack(">BB", major_tag | 24, length) + elif length < 65536: + return struct.pack(">BH", major_tag | 25, length) + elif length < 4294967296: + return struct.pack(">BL", major_tag | 26, length) + else: + return struct.pack(">BQ", major_tag | 27, length) + + +def encode_semantic(encoder, tag, value): + encoder.write(encode_length(0xC0, tag)) + encoder.encode(value) + + +def encode_float(encoder, value): + # Handle special values efficiently + import math + + if math.isnan(value): + encoder.write(b"\xf9\x7e\x00") + elif math.isinf(value): + encoder.write(b"\xf9\x7c\x00" if value > 0 else b"\xf9\xfc\x00") + else: + encoder.write(struct.pack(">Bd", 0xFB, value)) + + +def encode_int(encoder, value): + # Big integers (2 ** 64 and over) + if value >= 18446744073709551616 or value < -18446744073709551616: + if value >= 0: + major_type = 0x02 + else: + major_type = 0x03 + value = -value - 1 + + values = [] + while value > 0: + value, remainder = divmod(value, 256) + values.insert(0, remainder) + + payload = bytes(values) + encode_semantic(encoder, major_type, payload) + elif value >= 0: + encoder.write(encode_length(0, value)) + else: + encoder.write(encode_length(0x20, abs(value) - 1)) + + +def encode_bytestring(encoder, value): + encoder.write(encode_length(0x40, len(value)) + value) + + +def encode_bytearray(encoder, value): + encode_bytestring(encoder, bytes(value)) + + +def encode_string(encoder, value): + encoded = value.encode("utf-8") + encoder.write(encode_length(0x60, len(encoded)) + encoded) + + +def encode_map(encoder, value): + encoder.write(encode_length(0xA0, len(value))) + for key, val in value.items(): + encoder.encode(key) + encoder.encode(val) + + +def encode_array(encoder, value): + encoder.write(encode_length(0x80, len(value))) + for item in value: + encoder.encode(item) + + +def encode_boolean(encoder, value): + encoder.write(b"\xf5" if value else b"\xf4") + + +def encode_none(encoder, value): + encoder.write(b"\xf6") + + +cbor_encoders = { # supported data types and the encoder to use. + bytes: encode_bytestring, + bytearray: encode_bytearray, + str: encode_string, + int: encode_int, + float: encode_float, + bool: encode_boolean, + type(None): encode_none, + list: encode_array, + dict: encode_map, +} + + +class CBOREncoder(object): + """ + Serializes objects to a byte stream using Concise Binary Object Representation. + """ + + def __init__(self, fp): + self.fp = fp + + def _find_encoder(self, obj): + return cbor_encoders[type(obj)] + + def write(self, data): + """ + Write bytes to the data stream. + :param data: the bytes to write + """ + self.fp.write(data) + + def encode(self, obj): + """ + Encode the given object using CBOR. + :param obj: the object to encode + """ + encoder = self._find_encoder(obj) + if not encoder: + raise CBOREncodeError("cannot serialize type %s" % type(obj)) + encoder(self, obj) + + +def dumps(obj, **kwargs): + """ + Serialize an object to a bytestring. + :param obj: the object to serialize + :param kwargs: keyword arguments passed to :class:`~.CBOREncoder` + :return: the serialized output + :rtype: bytes + """ + fp = io.BytesIO() + dump(obj, fp, **kwargs) + return fp.getvalue() + + +def dump(obj, fp, **kwargs): + """ + Serialize an object to a file. + :param obj: the object to serialize + :param fp: a file-like object + :param kwargs: keyword arguments passed to :class:`~.CBOREncoder` + """ + CBOREncoder(fp, **kwargs).encode(obj) diff --git a/python-ecosys/cbor2/examples/cbor_test.py b/python-ecosys/cbor2/examples/cbor_test.py new file mode 100644 index 000000000..a1cd7e93e --- /dev/null +++ b/python-ecosys/cbor2/examples/cbor_test.py @@ -0,0 +1,37 @@ +""" +The MIT License (MIT) + +Copyright (c) 2023 Arduino SA +Copyright (c) 2018 KPN (Jan Bogaerts) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import cbor2 + +input = [ + {"bn": "urn:dev:ow:10e2073a01080063", "u": "Cel", "t": 1.276020076e09, "v": 23.5}, + {"u": "Cel", "t": 1.276020091e09, "v": 23.6}, +] + +data = cbor2.dumps(input) +print(data) +print(data.hex()) +text = cbor2.loads(data) +print(text) diff --git a/python-ecosys/cbor2/manifest.py b/python-ecosys/cbor2/manifest.py new file mode 100644 index 000000000..aa4b77092 --- /dev/null +++ b/python-ecosys/cbor2/manifest.py @@ -0,0 +1,3 @@ +metadata(version="1.0.0", pypi="cbor2") + +package("cbor2") diff --git a/python-ecosys/iperf3/iperf3.py b/python-ecosys/iperf3/iperf3.py index 59a4d6902..363d10d59 100644 --- a/python-ecosys/iperf3/iperf3.py +++ b/python-ecosys/iperf3/iperf3.py @@ -12,9 +12,22 @@ iperf3.client('192.168.1.5', udp=True, reverse=True) """ -import sys, os, struct -import time, select, socket import json +import select +import socket +import struct +import sys +import time + +# Provide a urandom() function, supporting devices without os.urandom(). +try: + from os import urandom +except ImportError: + from random import randint + + def urandom(n): + return bytes(randint(0, 255) for _ in range(n)) + DEBUG = False @@ -137,16 +150,19 @@ def stop(self): def report_receiver(self, stats): st = stats["streams"][0] - dt = st["end_time"] - st["start_time"] + + # iperf servers pre 3.2 do not transmit start or end time, + # so use local as fallback if not available. + dt = ticks_diff(self.t3, self.t0) + self.print_line( - st["start_time"], - st["end_time"], + st.get("start_time", 0.0), + st.get("end_time", dt * 1e-6), st["bytes"], st["packets"], st["errors"], " receiver", ) - return def recvn(s, n): @@ -177,7 +193,7 @@ def recvninto(s, buf): def make_cookie(): cookie_chars = b"abcdefghijklmnopqrstuvwxyz234567" cookie = bytearray(COOKIE_SIZE) - for i, x in enumerate(os.urandom(COOKIE_SIZE - 1)): + for i, x in enumerate(urandom(COOKIE_SIZE - 1)): cookie[i] = cookie_chars[x & 31] return cookie @@ -243,7 +259,7 @@ def server_once(): stats = Stats(param) stats.start() running = True - data_buf = bytearray(os.urandom(param["len"])) + data_buf = bytearray(urandom(param["len"])) while running: for pollable in poll.poll(stats.max_dt_ms()): if pollable_is_sock(pollable, s_ctrl): @@ -367,9 +383,11 @@ def client(host, udp=False, reverse=False, bandwidth=10 * 1024 * 1024): ticks_us_end = param["time"] * 1000000 poll = select.poll() poll.register(s_ctrl, select.POLLIN) + buf = None s_data = None start = None udp_packet_id = 0 + udp_last_send = None while True: for pollable in poll.poll(stats.max_dt_ms()): if pollable_is_sock(pollable, s_data): @@ -445,7 +463,7 @@ def client(host, udp=False, reverse=False, bandwidth=10 * 1024 * 1024): s_data = socket.socket(ai[0], socket.SOCK_STREAM) s_data.connect(ai[-1]) s_data.sendall(cookie) - buf = bytearray(os.urandom(param["len"])) + buf = bytearray(urandom(param["len"])) elif cmd == EXCHANGE_RESULTS: # Close data socket now that server knows we are finished, to prevent it flooding us poll.unregister(s_data) diff --git a/python-ecosys/iperf3/manifest.py b/python-ecosys/iperf3/manifest.py index dafba2e12..06964ce2a 100644 --- a/python-ecosys/iperf3/manifest.py +++ b/python-ecosys/iperf3/manifest.py @@ -1 +1,3 @@ +metadata(version="0.1.4", pypi="iperf3", pypi_publish="uiperf3") + module("iperf3.py") diff --git a/python-ecosys/pyjwt/manifest.py b/python-ecosys/pyjwt/manifest.py index 8e9b22c18..b3de5efc9 100644 --- a/python-ecosys/pyjwt/manifest.py +++ b/python-ecosys/pyjwt/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.1") +metadata(version="0.1.0", pypi="pyjwt") require("hmac") diff --git a/python-ecosys/requests/README.md b/python-ecosys/requests/README.md new file mode 100644 index 000000000..d6ceaadc5 --- /dev/null +++ b/python-ecosys/requests/README.md @@ -0,0 +1,16 @@ +## requests + +This module provides a lightweight version of the Python +[requests](https://requests.readthedocs.io/en/latest/) library. + +It includes support for all HTTP verbs, https, json decoding of responses, +redirects, basic authentication. + +### Limitations + +* Certificate validation is not currently supported. +* A dictionary passed as post data will not do automatic JSON or + multipart-form encoding of post data (this can be done manually). +* Compressed requests/responses are not currently supported. +* File upload is not supported. +* Chunked encoding in responses is not supported. diff --git a/python-ecosys/urequests/example_xively.py b/python-ecosys/requests/example_xively.py similarity index 80% rename from python-ecosys/urequests/example_xively.py rename to python-ecosys/requests/example_xively.py index 88b890cbc..60e139b98 100644 --- a/python-ecosys/urequests/example_xively.py +++ b/python-ecosys/requests/example_xively.py @@ -1,7 +1,4 @@ -try: - import urequests as requests -except ImportError: - import requests +import requests r = requests.get("http://api.xively.com/") print(r) diff --git a/python-ecosys/requests/manifest.py b/python-ecosys/requests/manifest.py new file mode 100644 index 000000000..85f159753 --- /dev/null +++ b/python-ecosys/requests/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.10.2", pypi="requests") + +package("requests") diff --git a/python-ecosys/urequests/urequests.py b/python-ecosys/requests/requests/__init__.py similarity index 70% rename from python-ecosys/urequests/urequests.py rename to python-ecosys/requests/requests/__init__.py index e1998711d..4ca7489a4 100644 --- a/python-ecosys/urequests/urequests.py +++ b/python-ecosys/requests/requests/__init__.py @@ -1,4 +1,4 @@ -import usocket +import socket class Response: @@ -28,9 +28,9 @@ def text(self): return str(self.content, self.encoding) def json(self): - import ujson + import json - return ujson.loads(self.content) + return json.loads(self.content) def request( @@ -38,21 +38,26 @@ def request( url, data=None, json=None, - headers={}, + headers=None, stream=None, auth=None, timeout=None, parse_headers=True, ): + if headers is None: + headers = {} + else: + headers = headers.copy() + redirect = None # redirection url, None means no redirection - chunked_data = data and getattr(data, "__iter__", None) and not getattr(data, "__len__", None) + chunked_data = data and getattr(data, "__next__", None) and not getattr(data, "__len__", None) if auth is not None: - import ubinascii + import binascii username, password = auth formated = b"{}:{}".format(username, password) - formated = str(ubinascii.b2a_base64(formated)[:-1], "ascii") + formated = str(binascii.b2a_base64(formated)[:-1], "ascii") headers["Authorization"] = "Basic {}".format(formated) try: @@ -63,7 +68,7 @@ def request( if proto == "http:": port = 80 elif proto == "https:": - import ussl + import tls port = 443 else: @@ -73,14 +78,14 @@ def request( host, port = host.split(":", 1) port = int(port) - ai = usocket.getaddrinfo(host, port, 0, usocket.SOCK_STREAM) + ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM) ai = ai[0] resp_d = None if parse_headers is not False: resp_d = {} - s = usocket.socket(ai[0], usocket.SOCK_STREAM, ai[2]) + s = socket.socket(ai[0], socket.SOCK_STREAM, ai[2]) if timeout is not None: # Note: settimeout is not supported on all platforms, will raise @@ -90,35 +95,53 @@ def request( try: s.connect(ai[-1]) if proto == "https:": - s = ussl.wrap_socket(s, server_hostname=host) + context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT) + context.verify_mode = tls.CERT_NONE + s = context.wrap_socket(s, server_hostname=host) s.write(b"%s /%s HTTP/1.0\r\n" % (method, path)) - if not "Host" in headers: - s.write(b"Host: %s\r\n" % host) + + if "Host" not in headers: + headers["Host"] = host + + if json is not None: + assert data is None + from json import dumps + + data = dumps(json) + + if "Content-Type" not in headers: + headers["Content-Type"] = "application/json" + + if data: + if chunked_data: + if "Transfer-Encoding" not in headers and "Content-Length" not in headers: + headers["Transfer-Encoding"] = "chunked" + elif "Content-Length" not in headers: + headers["Content-Length"] = str(len(data)) + + if "Connection" not in headers: + headers["Connection"] = "close" + # Iterate over keys to avoid tuple alloc for k in headers: s.write(k) s.write(b": ") s.write(headers[k]) s.write(b"\r\n") - if json is not None: - assert data is None - import ujson - data = ujson.dumps(json) - s.write(b"Content-Type: application/json\r\n") - if data: - if chunked_data: - s.write(b"Transfer-Encoding: chunked\r\n") - else: - s.write(b"Content-Length: %d\r\n" % len(data)) - s.write(b"Connection: close\r\n\r\n") + s.write(b"\r\n") + if data: if chunked_data: - for chunk in data: - s.write(b"%x\r\n" % len(chunk)) - s.write(chunk) - s.write(b"\r\n") - s.write("0\r\n\r\n") + if headers.get("Transfer-Encoding", None) == "chunked": + for chunk in data: + s.write(b"%x\r\n" % len(chunk)) + s.write(chunk) + s.write(b"\r\n") + s.write("0\r\n\r\n") + else: + for chunk in data: + s.write(chunk) else: s.write(data) @@ -159,6 +182,8 @@ def request( if redirect: s.close() + # Use the host specified in the redirect URL, as it may not be the same as the original URL. + headers.pop("Host", None) if status in [301, 302, 303]: return request("GET", redirect, None, None, headers, stream) else: diff --git a/python-ecosys/requests/test_requests.py b/python-ecosys/requests/test_requests.py new file mode 100644 index 000000000..ac77291b0 --- /dev/null +++ b/python-ecosys/requests/test_requests.py @@ -0,0 +1,164 @@ +import io +import sys + + +class Socket: + def __init__(self): + self._write_buffer = io.BytesIO() + self._read_buffer = io.BytesIO(b"HTTP/1.0 200 OK\r\n\r\n") + + def connect(self, address): + pass + + def write(self, buf): + self._write_buffer.write(buf) + + def readline(self): + return self._read_buffer.readline() + + +class socket: + AF_INET = 2 + SOCK_STREAM = 1 + IPPROTO_TCP = 6 + + @staticmethod + def getaddrinfo(host, port, af=0, type=0, flags=0): + return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("127.0.0.1", 80))] + + def socket(af=AF_INET, type=SOCK_STREAM, proto=IPPROTO_TCP): + return Socket() + + +sys.modules["socket"] = socket +# ruff: noqa: E402 +import requests + + +def format_message(response): + return response.raw._write_buffer.getvalue().decode("utf8") + + +def test_simple_get(): + response = requests.request("GET", "http://example.com/") + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + b"Connection: close\r\n" + b"Host: example.com\r\n\r\n" + ), format_message(response) + + +def test_get_auth(): + response = requests.request( + "GET", "http://example.com/", auth=("test-username", "test-password") + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Host: example.com\r\n" + + b"Authorization: Basic dGVzdC11c2VybmFtZTp0ZXN0LXBhc3N3b3Jk\r\n" + + b"Connection: close\r\n\r\n" + ), format_message(response) + + +def test_get_custom_header(): + response = requests.request("GET", "http://example.com/", headers={"User-Agent": "test-agent"}) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"User-Agent: test-agent\r\n" + + b"Host: example.com\r\n" + + b"Connection: close\r\n\r\n" + ), format_message(response) + + +def test_post_json(): + response = requests.request("GET", "http://example.com/", json="test") + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Connection: close\r\n" + + b"Content-Type: application/json\r\n" + + b"Host: example.com\r\n" + + b"Content-Length: 6\r\n\r\n" + + b'"test"' + ), format_message(response) + + +def test_post_chunked_data(): + def chunks(): + yield "test" + + response = requests.request("GET", "http://example.com/", data=chunks()) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Transfer-Encoding: chunked\r\n" + + b"Host: example.com\r\n" + + b"Connection: close\r\n\r\n" + + b"4\r\ntest\r\n" + + b"0\r\n\r\n" + ), format_message(response) + + +def test_overwrite_get_headers(): + response = requests.request( + "GET", "http://example.com/", headers={"Host": "test.com", "Connection": "keep-alive"} + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + b"Connection: keep-alive\r\n" + b"Host: test.com\r\n\r\n" + ), format_message(response) + + +def test_overwrite_post_json_headers(): + response = requests.request( + "GET", + "http://example.com/", + json="test", + headers={"Content-Type": "text/plain", "Content-Length": "10"}, + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Connection: close\r\n" + + b"Content-Length: 10\r\n" + + b"Content-Type: text/plain\r\n" + + b"Host: example.com\r\n\r\n" + + b'"test"' + ), format_message(response) + + +def test_overwrite_post_chunked_data_headers(): + def chunks(): + yield "test" + + response = requests.request( + "GET", "http://example.com/", data=chunks(), headers={"Content-Length": "4"} + ) + + assert response.raw._write_buffer.getvalue() == ( + b"GET / HTTP/1.0\r\n" + + b"Host: example.com\r\n" + + b"Content-Length: 4\r\n" + + b"Connection: close\r\n\r\n" + + b"test" + ), format_message(response) + + +def test_do_not_modify_headers_argument(): + global do_not_modify_this_dict + do_not_modify_this_dict = {} + requests.request("GET", "http://example.com/", headers=do_not_modify_this_dict) + + assert do_not_modify_this_dict == {}, do_not_modify_this_dict + + +test_simple_get() +test_get_auth() +test_get_custom_header() +test_post_json() +test_post_chunked_data() +test_overwrite_get_headers() +test_overwrite_post_json_headers() +test_overwrite_post_chunked_data_headers() +test_do_not_modify_headers_argument() diff --git a/python-ecosys/urequests/manifest.py b/python-ecosys/urequests/manifest.py deleted file mode 100644 index 5fd2e8a28..000000000 --- a/python-ecosys/urequests/manifest.py +++ /dev/null @@ -1,3 +0,0 @@ -metadata(version="0.7.0") - -module("urequests.py") diff --git a/python-stdlib/__future__/__future__.py b/python-stdlib/__future__/__future__.py index 45b935edc..178294c96 100644 --- a/python-stdlib/__future__/__future__.py +++ b/python-stdlib/__future__/__future__.py @@ -5,3 +5,4 @@ with_statement = True print_function = True unicode_literals = True +annotations = True diff --git a/python-stdlib/__future__/manifest.py b/python-stdlib/__future__/manifest.py index 4b4de03cb..e06f3268d 100644 --- a/python-stdlib/__future__/manifest.py +++ b/python-stdlib/__future__/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.0.3") +metadata(version="0.1.0") module("__future__.py") diff --git a/python-stdlib/argparse/argparse.py b/python-stdlib/argparse/argparse.py index cb575dd24..5c92887f9 100644 --- a/python-stdlib/argparse/argparse.py +++ b/python-stdlib/argparse/argparse.py @@ -3,7 +3,7 @@ """ import sys -from ucollections import namedtuple +from collections import namedtuple class _ArgError(BaseException): diff --git a/python-stdlib/argparse/manifest.py b/python-stdlib/argparse/manifest.py index 034d73c7d..02bf1a22c 100644 --- a/python-stdlib/argparse/manifest.py +++ b/python-stdlib/argparse/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.4") +metadata(version="0.4.0") # Originally written by Damien George. diff --git a/python-stdlib/base64/base64.py b/python-stdlib/base64/base64.py index daa39728b..d6baca05f 100644 --- a/python-stdlib/base64/base64.py +++ b/python-stdlib/base64/base64.py @@ -52,6 +52,25 @@ def _bytes_from_decode_data(s): raise TypeError("argument should be bytes or ASCII string, not %s" % s.__class__.__name__) +def _maketrans(f, t): + """Re-implement bytes.maketrans() as there is no such function in micropython""" + if len(f) != len(t): + raise ValueError("maketrans arguments must have same length") + translation_table = dict(zip(f, t)) + return translation_table + + +def _translate(input_bytes, trans_table): + """Re-implement bytes.translate() as there is no such function in micropython""" + result = bytearray() + + for byte in input_bytes: + translated_byte = trans_table.get(byte, byte) + result.append(translated_byte) + + return bytes(result) + + # Base64 encoding/decoding uses binascii @@ -73,7 +92,7 @@ def b64encode(s, altchars=None): if not isinstance(altchars, bytes_types): raise TypeError("expected bytes, not %s" % altchars.__class__.__name__) assert len(altchars) == 2, repr(altchars) - return encoded.translate(bytes.maketrans(b"+/", altchars)) + encoded = _translate(encoded, _maketrans(b"+/", altchars)) return encoded @@ -95,7 +114,7 @@ def b64decode(s, altchars=None, validate=False): if altchars is not None: altchars = _bytes_from_decode_data(altchars) assert len(altchars) == 2, repr(altchars) - s = s.translate(bytes.maketrans(altchars, b"+/")) + s = _translate(s, _maketrans(altchars, b"+/")) if validate and not re.match(b"^[A-Za-z0-9+/]*=*$", s): raise binascii.Error("Non-base64 digit found") return binascii.a2b_base64(s) @@ -120,8 +139,8 @@ def standard_b64decode(s): return b64decode(s) -# _urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_') -# _urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/') +# _urlsafe_encode_translation = _maketrans(b'+/', b'-_') +# _urlsafe_decode_translation = _maketrans(b'-_', b'+/') def urlsafe_b64encode(s): @@ -132,7 +151,7 @@ def urlsafe_b64encode(s): '/'. """ # return b64encode(s).translate(_urlsafe_encode_translation) - raise NotImplementedError() + return b64encode(s, b"-_").rstrip(b"\n") def urlsafe_b64decode(s): @@ -266,7 +285,7 @@ def b32decode(s, casefold=False, map01=None): if map01 is not None: map01 = _bytes_from_decode_data(map01) assert len(map01) == 1, repr(map01) - s = s.translate(bytes.maketrans(b"01", b"O" + map01)) + s = _translate(s, _maketrans(b"01", b"O" + map01)) if casefold: s = s.upper() # Strip off pad characters from the right. We need to count the pad diff --git a/python-stdlib/base64/manifest.py b/python-stdlib/base64/manifest.py index 59d39f78b..9e1b31751 100644 --- a/python-stdlib/base64/manifest.py +++ b/python-stdlib/base64/manifest.py @@ -1,6 +1,5 @@ -metadata(version="3.3.3-4") +metadata(version="3.3.6") require("binascii") -require("struct") module("base64.py") diff --git a/python-stdlib/binascii/manifest.py b/python-stdlib/binascii/manifest.py index 4a478f262..c637678a1 100644 --- a/python-stdlib/binascii/manifest.py +++ b/python-stdlib/binascii/manifest.py @@ -1,3 +1,3 @@ -metadata(version="2.4.0-5") +metadata(version="2.4.1") module("binascii.py") diff --git a/python-stdlib/binascii/test_binascii.py b/python-stdlib/binascii/test_binascii.py index 942ddc51b..075b2ff3c 100644 --- a/python-stdlib/binascii/test_binascii.py +++ b/python-stdlib/binascii/test_binascii.py @@ -1,5 +1,5 @@ from binascii import * -import utime +import time data = b"zlutoucky kun upel dabelske ody" h = hexlify(data) @@ -14,10 +14,10 @@ a2b_base64(b"as==") == b"j" -start = utime.time() +start = time.time() for x in range(100000): d = unhexlify(h) -print("100000 iterations in: " + str(utime.time() - start)) +print("100000 iterations in: " + str(time.time() - start)) print("OK") diff --git a/python-stdlib/bisect/manifest.py b/python-stdlib/bisect/manifest.py new file mode 100644 index 000000000..5ba5a9a6b --- /dev/null +++ b/python-stdlib/bisect/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.5.0") + +module("bisect.py") diff --git a/python-stdlib/cmd/cmd.py b/python-stdlib/cmd/cmd.py index 9d46a8a7d..447ea1489 100644 --- a/python-stdlib/cmd/cmd.py +++ b/python-stdlib/cmd/cmd.py @@ -51,13 +51,12 @@ completions have also been stripped out. """ -# import string, sys -import sys # MiroPython doesn't yet have a string module +import sys __all__ = ["Cmd"] PROMPT = "(Cmd) " -# IDENTCHARS = string.ascii_letters + string.digits + '_' +# This is equivalent to string.ascii_letters + string.digits + '_' IDENTCHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_" diff --git a/python-stdlib/cmd/manifest.py b/python-stdlib/cmd/manifest.py index 572f97df6..910352ee7 100644 --- a/python-stdlib/cmd/manifest.py +++ b/python-stdlib/cmd/manifest.py @@ -1,3 +1,3 @@ -metadata(version="3.4.0-2") +metadata(version="3.4.1") module("cmd.py") diff --git a/python-stdlib/collections-defaultdict/manifest.py b/python-stdlib/collections-defaultdict/manifest.py index f8d566aba..e5c06e668 100644 --- a/python-stdlib/collections-defaultdict/manifest.py +++ b/python-stdlib/collections-defaultdict/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.3") +metadata(version="0.3.0") # Originally written by Paul Sokolovsky. diff --git a/python-stdlib/collections-deque/collections/deque.py b/python-stdlib/collections-deque/collections/deque.py deleted file mode 100644 index 1d8c62d4b..000000000 --- a/python-stdlib/collections-deque/collections/deque.py +++ /dev/null @@ -1,36 +0,0 @@ -class deque: - def __init__(self, iterable=None): - if iterable is None: - self.q = [] - else: - self.q = list(iterable) - - def popleft(self): - return self.q.pop(0) - - def popright(self): - return self.q.pop() - - def pop(self): - return self.q.pop() - - def append(self, a): - self.q.append(a) - - def appendleft(self, a): - self.q.insert(0, a) - - def extend(self, a): - self.q.extend(a) - - def __len__(self): - return len(self.q) - - def __bool__(self): - return bool(self.q) - - def __iter__(self): - yield from self.q - - def __str__(self): - return "deque({})".format(self.q) diff --git a/python-stdlib/collections-deque/manifest.py b/python-stdlib/collections-deque/manifest.py deleted file mode 100644 index 0133d2bad..000000000 --- a/python-stdlib/collections-deque/manifest.py +++ /dev/null @@ -1,4 +0,0 @@ -metadata(version="0.1.3") - -require("collections") -package("collections") diff --git a/python-stdlib/collections/collections/__init__.py b/python-stdlib/collections/collections/__init__.py index 7f3be5673..36dfc1c41 100644 --- a/python-stdlib/collections/collections/__init__.py +++ b/python-stdlib/collections/collections/__init__.py @@ -6,10 +6,6 @@ from .defaultdict import defaultdict except ImportError: pass -try: - from .deque import deque -except ImportError: - pass class MutableMapping: diff --git a/python-stdlib/collections/manifest.py b/python-stdlib/collections/manifest.py index d5ef69472..0ce56d1fa 100644 --- a/python-stdlib/collections/manifest.py +++ b/python-stdlib/collections/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.2") +metadata(version="0.2.0") package("collections") diff --git a/python-stdlib/contextlib/contextlib.py b/python-stdlib/contextlib/contextlib.py index 2b2020357..3e598b4b6 100644 --- a/python-stdlib/contextlib/contextlib.py +++ b/python-stdlib/contextlib/contextlib.py @@ -85,13 +85,13 @@ class ExitStack(object): """ def __init__(self): - self._exit_callbacks = deque() + self._exit_callbacks = [] def pop_all(self): """Preserve the context stack by transferring it to a new instance""" new_stack = type(self)() new_stack._exit_callbacks = self._exit_callbacks - self._exit_callbacks = deque() + self._exit_callbacks = [] return new_stack def _push_cm_exit(self, cm, cm_exit): diff --git a/python-stdlib/contextlib/manifest.py b/python-stdlib/contextlib/manifest.py index ab7ae5775..3e05bca18 100644 --- a/python-stdlib/contextlib/manifest.py +++ b/python-stdlib/contextlib/manifest.py @@ -1,4 +1,4 @@ -metadata(description="Port of contextlib for micropython", version="3.4.2-4") +metadata(description="Port of contextlib for micropython", version="3.4.4") require("ucontextlib") require("collections") diff --git a/python-stdlib/contextlib/tests.py b/python-stdlib/contextlib/tests.py index 19f07add8..c122c452e 100644 --- a/python-stdlib/contextlib/tests.py +++ b/python-stdlib/contextlib/tests.py @@ -399,7 +399,7 @@ def test_exit_exception_chaining_suppress(self): def test_excessive_nesting(self): # The original implementation would die with RecursionError here with ExitStack() as stack: - for i in range(10000): + for i in range(5000): stack.callback(int) def test_instance_bypass(self): diff --git a/python-stdlib/copy/copy.py b/python-stdlib/copy/copy.py index f7bfdd6a1..0a9283777 100644 --- a/python-stdlib/copy/copy.py +++ b/python-stdlib/copy/copy.py @@ -62,7 +62,7 @@ class Error(Exception): error = Error # backward compatibility try: - from ucollections import OrderedDict + from collections import OrderedDict except ImportError: OrderedDict = None diff --git a/python-stdlib/copy/manifest.py b/python-stdlib/copy/manifest.py index 909ac2054..b22ebeb90 100644 --- a/python-stdlib/copy/manifest.py +++ b/python-stdlib/copy/manifest.py @@ -1,3 +1,5 @@ -metadata(version="3.3.3-2") +metadata(version="3.3.4") + +require("types") module("copy.py") diff --git a/python-stdlib/curses.ascii/manifest.py b/python-stdlib/curses.ascii/manifest.py index 6a6518089..643e3d49a 100644 --- a/python-stdlib/curses.ascii/manifest.py +++ b/python-stdlib/curses.ascii/manifest.py @@ -1,3 +1,3 @@ -metadata(version="3.4.2-1") +metadata(version="3.4.3") package("curses") diff --git a/python-stdlib/datetime/datetime.py b/python-stdlib/datetime/datetime.py index b3cd9b94f..0f2a89105 100644 --- a/python-stdlib/datetime/datetime.py +++ b/python-stdlib/datetime/datetime.py @@ -2,8 +2,6 @@ import time as _tmod -__version__ = "2.0.0" - _DBM = (0, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334) _DIM = (0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31) _TIME_SPEC = ("auto", "hours", "minutes", "seconds", "milliseconds", "microseconds") diff --git a/python-stdlib/datetime/test_datetime.py b/python-stdlib/datetime/test_datetime.py index 372bdf3de..98da458f9 100644 --- a/python-stdlib/datetime/test_datetime.py +++ b/python-stdlib/datetime/test_datetime.py @@ -2082,9 +2082,11 @@ def test_timetuple00(self): with LocalTz("Europe/Rome"): self.assertEqual(dt1.timetuple()[:8], (2002, 1, 31, 0, 0, 0, 3, 31)) + @unittest.skip("broken when running with non-UTC timezone") def test_timetuple01(self): self.assertEqual(dt27tz2.timetuple()[:8], (2010, 3, 27, 12, 0, 0, 5, 86)) + @unittest.skip("broken when running with non-UTC timezone") def test_timetuple02(self): self.assertEqual(dt28tz2.timetuple()[:8], (2010, 3, 28, 12, 0, 0, 6, 87)) diff --git a/python-stdlib/fnmatch/fnmatch.py b/python-stdlib/fnmatch/fnmatch.py index 71009afa2..2b42c3be4 100644 --- a/python-stdlib/fnmatch/fnmatch.py +++ b/python-stdlib/fnmatch/fnmatch.py @@ -27,8 +27,6 @@ def normcase(s): __all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] -COMPAT = re.__name__ == "ure" - def fnmatch(name, pat): """Test whether FILENAME matches PATTERN. @@ -58,12 +56,18 @@ def _compile_pattern(pat): res = bytes(res_str, "ISO-8859-1") else: res = translate(pat) - if COMPAT: + + try: + ptn = re.compile(res) + except ValueError: + # re1.5 doesn't support all regex features if res.startswith("(?ms)"): res = res[5:] if res.endswith("\\Z"): res = res[:-2] + "$" - return re.compile(res).match + ptn = re.compile(res) + + return ptn.match def filter(names, pat): diff --git a/python-stdlib/fnmatch/manifest.py b/python-stdlib/fnmatch/manifest.py index 8f19bb8f4..f4318b374 100644 --- a/python-stdlib/fnmatch/manifest.py +++ b/python-stdlib/fnmatch/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.6.0") +metadata(version="0.6.1") module("fnmatch.py") diff --git a/python-stdlib/fnmatch/test_fnmatch.py b/python-stdlib/fnmatch/test_fnmatch.py index 4eaeec63b..97ef8fff7 100644 --- a/python-stdlib/fnmatch/test_fnmatch.py +++ b/python-stdlib/fnmatch/test_fnmatch.py @@ -1,6 +1,5 @@ """Test cases for the fnmatch module.""" -from test import support import unittest from fnmatch import fnmatch, fnmatchcase, translate, filter @@ -79,11 +78,3 @@ def test_translate(self): class FilterTestCase(unittest.TestCase): def test_filter(self): self.assertEqual(filter(["a", "b"], "a"), ["a"]) - - -def main(): - support.run_unittest(FnmatchTestCase, TranslateTestCase, FilterTestCase) - - -if __name__ == "__main__": - main() diff --git a/python-stdlib/gzip/gzip.py b/python-stdlib/gzip/gzip.py index 6d6c967a6..12bfb1ff5 100644 --- a/python-stdlib/gzip/gzip.py +++ b/python-stdlib/gzip/gzip.py @@ -1,29 +1,29 @@ -# import zlib -import uzlib as zlib +# MicroPython gzip module +# MIT license; Copyright (c) 2023 Jim Mussared -FTEXT = 1 -FHCRC = 2 -FEXTRA = 4 -FNAME = 8 -FCOMMENT = 16 +_WBITS = const(15) + +import builtins, io, deflate + + +def GzipFile(fileobj): + return deflate.DeflateIO(fileobj, deflate.GZIP, _WBITS) + + +def open(filename, mode="rb"): + return deflate.DeflateIO(builtins.open(filename, mode), deflate.GZIP, _WBITS, True) + + +if hasattr(deflate.DeflateIO, "write"): + + def compress(data): + f = io.BytesIO() + with GzipFile(fileobj=f) as g: + g.write(data) + return f.getvalue() def decompress(data): - assert data[0] == 0x1F and data[1] == 0x8B - assert data[2] == 8 - flg = data[3] - assert flg & 0xE0 == 0 - i = 10 - if flg & FEXTRA: - i += data[11] << 8 + data[10] + 2 - if flg & FNAME: - while data[i]: - i += 1 - i += 1 - if flg & FCOMMENT: - while data[i]: - i += 1 - i += 1 - if flg & FHCRC: - i += 2 - return zlib.decompress(memoryview(data)[i:], -15) + f = io.BytesIO(data) + with GzipFile(fileobj=f) as g: + return g.read() diff --git a/python-stdlib/gzip/manifest.py b/python-stdlib/gzip/manifest.py index eab70e56b..c422b2965 100644 --- a/python-stdlib/gzip/manifest.py +++ b/python-stdlib/gzip/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.1") +metadata(version="1.0.1") module("gzip.py") diff --git a/python-stdlib/hashlib-core/hashlib/__init__.py b/python-stdlib/hashlib-core/hashlib/__init__.py new file mode 100644 index 000000000..932b6f647 --- /dev/null +++ b/python-stdlib/hashlib-core/hashlib/__init__.py @@ -0,0 +1,29 @@ +# Use built-in algorithms preferentially (on many ports this is just sha256). +try: + from uhashlib import * +except ImportError: + pass + + +# Add missing algorithms based on installed extensions. +def _init(): + for algo in ("sha224", "sha256", "sha384", "sha512"): + if algo not in globals(): + try: + # from ._{algo} import {algo} + c = __import__("_" + algo, None, None, (), 1) + globals()[algo] = getattr(c, algo) + except ImportError: + pass + + +_init() +del _init + + +def new(algo, data=b""): + try: + c = globals()[algo] + return c(data) + except KeyError: + raise ValueError(algo) diff --git a/python-stdlib/hashlib-core/hashlib/_sha.py b/python-stdlib/hashlib-core/hashlib/_sha.py new file mode 100644 index 000000000..4e7339c76 --- /dev/null +++ b/python-stdlib/hashlib-core/hashlib/_sha.py @@ -0,0 +1,42 @@ +# MIT license; Copyright (c) 2023 Jim Mussared +# Originally ported from CPython by Paul Sokolovsky + + +# Base class for SHA implementations, which must provide: +# .digestsize & .digest_size +# .block_size +# ._iv +# ._update +# ._final +class sha: + def __init__(self, s=None): + self._digest = self._iv[:] + self._count_lo = 0 + self._count_hi = 0 + self._data = bytearray(self.block_size) + self._local = 0 + self._digestsize = self.digest_size + if s: + self.update(s) + + def update(self, s): + if isinstance(s, str): + s = s.encode("ascii") + else: + s = bytes(s) + self._update(s) + + def digest(self): + return self.copy()._final()[: self._digestsize] + + def hexdigest(self): + return "".join(["%.2x" % i for i in self.digest()]) + + def copy(self): + new = type(self)() + new._digest = self._digest[:] + new._count_lo = self._count_lo + new._count_hi = self._count_hi + new._data = self._data[:] + new._local = self._local + return new diff --git a/python-stdlib/hashlib-core/manifest.py b/python-stdlib/hashlib-core/manifest.py new file mode 100644 index 000000000..db8d42482 --- /dev/null +++ b/python-stdlib/hashlib-core/manifest.py @@ -0,0 +1,3 @@ +metadata(version="1.0.0") + +package("hashlib") diff --git a/python-stdlib/hashlib-sha224/hashlib/_sha224.py b/python-stdlib/hashlib-sha224/hashlib/_sha224.py new file mode 100644 index 000000000..4f6dc7181 --- /dev/null +++ b/python-stdlib/hashlib-sha224/hashlib/_sha224.py @@ -0,0 +1,18 @@ +# MIT license; Copyright (c) 2023 Jim Mussared +# Originally ported from CPython by Paul Sokolovsky + +from ._sha256 import sha256 + + +class sha224(sha256): + digest_size = digestsize = 28 + _iv = [ + 0xC1059ED8, + 0x367CD507, + 0x3070DD17, + 0xF70E5939, + 0xFFC00B31, + 0x68581511, + 0x64F98FA7, + 0xBEFA4FA4, + ] diff --git a/python-stdlib/hashlib-sha224/manifest.py b/python-stdlib/hashlib-sha224/manifest.py new file mode 100644 index 000000000..30a2c2531 --- /dev/null +++ b/python-stdlib/hashlib-sha224/manifest.py @@ -0,0 +1,4 @@ +metadata(version="1.0.0", description="Adds the SHA224 hash algorithm to hashlib.") + +require("hashlib-sha256") +package("hashlib") diff --git a/python-stdlib/hashlib-sha256/hashlib/_sha256.py b/python-stdlib/hashlib-sha256/hashlib/_sha256.py new file mode 100644 index 000000000..43fc1522b --- /dev/null +++ b/python-stdlib/hashlib-sha256/hashlib/_sha256.py @@ -0,0 +1,194 @@ +# MIT license; Copyright (c) 2023 Jim Mussared +# Originally ported from CPython by Paul Sokolovsky + +from ._sha import sha + +_SHA_BLOCKSIZE = const(64) + + +ROR = lambda x, y: (((x & 0xFFFFFFFF) >> (y & 31)) | (x << (32 - (y & 31)))) & 0xFFFFFFFF +Ch = lambda x, y, z: (z ^ (x & (y ^ z))) +Maj = lambda x, y, z: (((x | y) & z) | (x & y)) +S = lambda x, n: ROR(x, n) +R = lambda x, n: (x & 0xFFFFFFFF) >> n +Sigma0 = lambda x: (S(x, 2) ^ S(x, 13) ^ S(x, 22)) +Sigma1 = lambda x: (S(x, 6) ^ S(x, 11) ^ S(x, 25)) +Gamma0 = lambda x: (S(x, 7) ^ S(x, 18) ^ R(x, 3)) +Gamma1 = lambda x: (S(x, 17) ^ S(x, 19) ^ R(x, 10)) + + +class sha256(sha): + digest_size = digestsize = 32 + block_size = _SHA_BLOCKSIZE + _iv = [ + 0x6A09E667, + 0xBB67AE85, + 0x3C6EF372, + 0xA54FF53A, + 0x510E527F, + 0x9B05688C, + 0x1F83D9AB, + 0x5BE0CD19, + ] + + def _transform(self): + W = [] + + d = self._data + for i in range(0, 16): + W.append((d[4 * i] << 24) + (d[4 * i + 1] << 16) + (d[4 * i + 2] << 8) + d[4 * i + 3]) + + for i in range(16, 64): + W.append((Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xFFFFFFFF) + + ss = self._digest[:] + + def RND(a, b, c, d, e, f, g, h, i, ki): + t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i] + t1 = Sigma0(a) + Maj(a, b, c) + d += t0 + h = t0 + t1 + return d & 0xFFFFFFFF, h & 0xFFFFFFFF + + ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 0, 0x428A2F98) + ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 1, 0x71374491) + ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 2, 0xB5C0FBCF) + ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 3, 0xE9B5DBA5) + ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 4, 0x3956C25B) + ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 5, 0x59F111F1) + ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 6, 0x923F82A4) + ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 7, 0xAB1C5ED5) + ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 8, 0xD807AA98) + ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 9, 0x12835B01) + ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 10, 0x243185BE) + ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 11, 0x550C7DC3) + ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 12, 0x72BE5D74) + ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 13, 0x80DEB1FE) + ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 14, 0x9BDC06A7) + ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 15, 0xC19BF174) + ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 16, 0xE49B69C1) + ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 17, 0xEFBE4786) + ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 18, 0x0FC19DC6) + ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 19, 0x240CA1CC) + ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 20, 0x2DE92C6F) + ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 21, 0x4A7484AA) + ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 22, 0x5CB0A9DC) + ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 23, 0x76F988DA) + ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 24, 0x983E5152) + ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 25, 0xA831C66D) + ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 26, 0xB00327C8) + ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 27, 0xBF597FC7) + ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 28, 0xC6E00BF3) + ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 29, 0xD5A79147) + ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 30, 0x06CA6351) + ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 31, 0x14292967) + ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 32, 0x27B70A85) + ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 33, 0x2E1B2138) + ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 34, 0x4D2C6DFC) + ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 35, 0x53380D13) + ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 36, 0x650A7354) + ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 37, 0x766A0ABB) + ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 38, 0x81C2C92E) + ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 39, 0x92722C85) + ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 40, 0xA2BFE8A1) + ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 41, 0xA81A664B) + ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 42, 0xC24B8B70) + ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 43, 0xC76C51A3) + ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 44, 0xD192E819) + ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 45, 0xD6990624) + ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 46, 0xF40E3585) + ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 47, 0x106AA070) + ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 48, 0x19A4C116) + ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 49, 0x1E376C08) + ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 50, 0x2748774C) + ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 51, 0x34B0BCB5) + ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 52, 0x391C0CB3) + ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 53, 0x4ED8AA4A) + ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 54, 0x5B9CCA4F) + ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 55, 0x682E6FF3) + ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 56, 0x748F82EE) + ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 57, 0x78A5636F) + ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 58, 0x84C87814) + ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 59, 0x8CC70208) + ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 60, 0x90BEFFFA) + ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 61, 0xA4506CEB) + ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 62, 0xBEF9A3F7) + ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 63, 0xC67178F2) + + for i in range(len(self._digest)): + self._digest[i] = (self._digest[i] + ss[i]) & 0xFFFFFFFF + + def _update(self, buffer): + if isinstance(buffer, str): + raise TypeError("Unicode strings must be encoded before hashing") + count = len(buffer) + buffer_idx = 0 + clo = (self._count_lo + (count << 3)) & 0xFFFFFFFF + if clo < self._count_lo: + self._count_hi += 1 + self._count_lo = clo + + self._count_hi += count >> 29 + + if self._local: + i = _SHA_BLOCKSIZE - self._local + if i > count: + i = count + + # copy buffer + for x in enumerate(buffer[buffer_idx : buffer_idx + i]): + self._data[self._local + x[0]] = x[1] + + count -= i + buffer_idx += i + + self._local += i + if self._local == _SHA_BLOCKSIZE: + self._transform() + self._local = 0 + else: + return + + while count >= _SHA_BLOCKSIZE: + # copy buffer + self._data = bytearray(buffer[buffer_idx : buffer_idx + _SHA_BLOCKSIZE]) + count -= _SHA_BLOCKSIZE + buffer_idx += _SHA_BLOCKSIZE + self._transform() + + # copy buffer + pos = self._local + self._data[pos : pos + count] = buffer[buffer_idx : buffer_idx + count] + self._local = count + + def _final(self): + lo_bit_count = self._count_lo + hi_bit_count = self._count_hi + count = (lo_bit_count >> 3) & 0x3F + self._data[count] = 0x80 + count += 1 + if count > _SHA_BLOCKSIZE - 8: + # zero the bytes in data after the count + self._data = self._data[:count] + bytes(_SHA_BLOCKSIZE - count) + self._transform() + # zero bytes in data + self._data = bytearray(_SHA_BLOCKSIZE) + else: + self._data = self._data[:count] + bytes(_SHA_BLOCKSIZE - count) + + self._data[56] = (hi_bit_count >> 24) & 0xFF + self._data[57] = (hi_bit_count >> 16) & 0xFF + self._data[58] = (hi_bit_count >> 8) & 0xFF + self._data[59] = (hi_bit_count >> 0) & 0xFF + self._data[60] = (lo_bit_count >> 24) & 0xFF + self._data[61] = (lo_bit_count >> 16) & 0xFF + self._data[62] = (lo_bit_count >> 8) & 0xFF + self._data[63] = (lo_bit_count >> 0) & 0xFF + + self._transform() + + dig = bytearray() + for i in self._digest: + for j in range(4): + dig.append((i >> ((3 - j) * 8)) & 0xFF) + return dig diff --git a/python-stdlib/hashlib-sha256/manifest.py b/python-stdlib/hashlib-sha256/manifest.py new file mode 100644 index 000000000..42a859e1d --- /dev/null +++ b/python-stdlib/hashlib-sha256/manifest.py @@ -0,0 +1,4 @@ +metadata(version="1.0.0", description="Adds the SHA256 hash algorithm to hashlib.") + +require("hashlib-core") +package("hashlib") diff --git a/python-stdlib/hashlib-sha384/hashlib/_sha384.py b/python-stdlib/hashlib-sha384/hashlib/_sha384.py new file mode 100644 index 000000000..fe15a10af --- /dev/null +++ b/python-stdlib/hashlib-sha384/hashlib/_sha384.py @@ -0,0 +1,18 @@ +# MIT license; Copyright (c) 2023 Jim Mussared +# Originally ported from CPython by Paul Sokolovsky + +from ._sha512 import sha512 + + +class sha384(sha512): + digest_size = digestsize = 48 + _iv = [ + 0xCBBB9D5DC1059ED8, + 0x629A292A367CD507, + 0x9159015A3070DD17, + 0x152FECD8F70E5939, + 0x67332667FFC00B31, + 0x8EB44A8768581511, + 0xDB0C2E0D64F98FA7, + 0x47B5481DBEFA4FA4, + ] diff --git a/python-stdlib/hashlib-sha384/manifest.py b/python-stdlib/hashlib-sha384/manifest.py new file mode 100644 index 000000000..6791eb56c --- /dev/null +++ b/python-stdlib/hashlib-sha384/manifest.py @@ -0,0 +1,4 @@ +metadata(version="1.0.0", description="Adds the SHA384 hash algorithm to hashlib.") + +require("hashlib-sha512") +package("hashlib") diff --git a/python-stdlib/hashlib-sha512/hashlib/_sha512.py b/python-stdlib/hashlib-sha512/hashlib/_sha512.py new file mode 100644 index 000000000..44e8656a6 --- /dev/null +++ b/python-stdlib/hashlib-sha512/hashlib/_sha512.py @@ -0,0 +1,393 @@ +# MIT license; Copyright (c) 2023 Jim Mussared +# Originally ported from CPython by Paul Sokolovsky + +from ._sha import sha + +_SHA_BLOCKSIZE = const(128) + + +ROR64 = ( + lambda x, y: (((x & 0xFFFFFFFFFFFFFFFF) >> (y & 63)) | (x << (64 - (y & 63)))) + & 0xFFFFFFFFFFFFFFFF +) +Ch = lambda x, y, z: (z ^ (x & (y ^ z))) +Maj = lambda x, y, z: (((x | y) & z) | (x & y)) +S = lambda x, n: ROR64(x, n) +R = lambda x, n: (x & 0xFFFFFFFFFFFFFFFF) >> n +Sigma0 = lambda x: (S(x, 28) ^ S(x, 34) ^ S(x, 39)) +Sigma1 = lambda x: (S(x, 14) ^ S(x, 18) ^ S(x, 41)) +Gamma0 = lambda x: (S(x, 1) ^ S(x, 8) ^ R(x, 7)) +Gamma1 = lambda x: (S(x, 19) ^ S(x, 61) ^ R(x, 6)) + + +class sha512(sha): + digest_size = digestsize = 64 + block_size = _SHA_BLOCKSIZE + _iv = [ + 0x6A09E667F3BCC908, + 0xBB67AE8584CAA73B, + 0x3C6EF372FE94F82B, + 0xA54FF53A5F1D36F1, + 0x510E527FADE682D1, + 0x9B05688C2B3E6C1F, + 0x1F83D9ABFB41BD6B, + 0x5BE0CD19137E2179, + ] + + def _transform(self): + W = [] + + d = self._data + for i in range(0, 16): + W.append( + (d[8 * i] << 56) + + (d[8 * i + 1] << 48) + + (d[8 * i + 2] << 40) + + (d[8 * i + 3] << 32) + + (d[8 * i + 4] << 24) + + (d[8 * i + 5] << 16) + + (d[8 * i + 6] << 8) + + d[8 * i + 7] + ) + + for i in range(16, 80): + W.append( + (Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xFFFFFFFFFFFFFFFF + ) + + ss = self._digest[:] + + def RND(a, b, c, d, e, f, g, h, i, ki): + t0 = (h + Sigma1(e) + Ch(e, f, g) + ki + W[i]) & 0xFFFFFFFFFFFFFFFF + t1 = (Sigma0(a) + Maj(a, b, c)) & 0xFFFFFFFFFFFFFFFF + d = (d + t0) & 0xFFFFFFFFFFFFFFFF + h = (t0 + t1) & 0xFFFFFFFFFFFFFFFF + return d & 0xFFFFFFFFFFFFFFFF, h & 0xFFFFFFFFFFFFFFFF + + ss[3], ss[7] = RND( + ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 0, 0x428A2F98D728AE22 + ) + ss[2], ss[6] = RND( + ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 1, 0x7137449123EF65CD + ) + ss[1], ss[5] = RND( + ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 2, 0xB5C0FBCFEC4D3B2F + ) + ss[0], ss[4] = RND( + ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 3, 0xE9B5DBA58189DBBC + ) + ss[7], ss[3] = RND( + ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 4, 0x3956C25BF348B538 + ) + ss[6], ss[2] = RND( + ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 5, 0x59F111F1B605D019 + ) + ss[5], ss[1] = RND( + ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 6, 0x923F82A4AF194F9B + ) + ss[4], ss[0] = RND( + ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 7, 0xAB1C5ED5DA6D8118 + ) + ss[3], ss[7] = RND( + ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 8, 0xD807AA98A3030242 + ) + ss[2], ss[6] = RND( + ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 9, 0x12835B0145706FBE + ) + ss[1], ss[5] = RND( + ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 10, 0x243185BE4EE4B28C + ) + ss[0], ss[4] = RND( + ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 11, 0x550C7DC3D5FFB4E2 + ) + ss[7], ss[3] = RND( + ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 12, 0x72BE5D74F27B896F + ) + ss[6], ss[2] = RND( + ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 13, 0x80DEB1FE3B1696B1 + ) + ss[5], ss[1] = RND( + ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 14, 0x9BDC06A725C71235 + ) + ss[4], ss[0] = RND( + ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 15, 0xC19BF174CF692694 + ) + ss[3], ss[7] = RND( + ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 16, 0xE49B69C19EF14AD2 + ) + ss[2], ss[6] = RND( + ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 17, 0xEFBE4786384F25E3 + ) + ss[1], ss[5] = RND( + ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 18, 0x0FC19DC68B8CD5B5 + ) + ss[0], ss[4] = RND( + ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 19, 0x240CA1CC77AC9C65 + ) + ss[7], ss[3] = RND( + ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 20, 0x2DE92C6F592B0275 + ) + ss[6], ss[2] = RND( + ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 21, 0x4A7484AA6EA6E483 + ) + ss[5], ss[1] = RND( + ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 22, 0x5CB0A9DCBD41FBD4 + ) + ss[4], ss[0] = RND( + ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 23, 0x76F988DA831153B5 + ) + ss[3], ss[7] = RND( + ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 24, 0x983E5152EE66DFAB + ) + ss[2], ss[6] = RND( + ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 25, 0xA831C66D2DB43210 + ) + ss[1], ss[5] = RND( + ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 26, 0xB00327C898FB213F + ) + ss[0], ss[4] = RND( + ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 27, 0xBF597FC7BEEF0EE4 + ) + ss[7], ss[3] = RND( + ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 28, 0xC6E00BF33DA88FC2 + ) + ss[6], ss[2] = RND( + ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 29, 0xD5A79147930AA725 + ) + ss[5], ss[1] = RND( + ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 30, 0x06CA6351E003826F + ) + ss[4], ss[0] = RND( + ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 31, 0x142929670A0E6E70 + ) + ss[3], ss[7] = RND( + ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 32, 0x27B70A8546D22FFC + ) + ss[2], ss[6] = RND( + ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 33, 0x2E1B21385C26C926 + ) + ss[1], ss[5] = RND( + ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 34, 0x4D2C6DFC5AC42AED + ) + ss[0], ss[4] = RND( + ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 35, 0x53380D139D95B3DF + ) + ss[7], ss[3] = RND( + ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 36, 0x650A73548BAF63DE + ) + ss[6], ss[2] = RND( + ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 37, 0x766A0ABB3C77B2A8 + ) + ss[5], ss[1] = RND( + ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 38, 0x81C2C92E47EDAEE6 + ) + ss[4], ss[0] = RND( + ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 39, 0x92722C851482353B + ) + ss[3], ss[7] = RND( + ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 40, 0xA2BFE8A14CF10364 + ) + ss[2], ss[6] = RND( + ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 41, 0xA81A664BBC423001 + ) + ss[1], ss[5] = RND( + ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 42, 0xC24B8B70D0F89791 + ) + ss[0], ss[4] = RND( + ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 43, 0xC76C51A30654BE30 + ) + ss[7], ss[3] = RND( + ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 44, 0xD192E819D6EF5218 + ) + ss[6], ss[2] = RND( + ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 45, 0xD69906245565A910 + ) + ss[5], ss[1] = RND( + ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 46, 0xF40E35855771202A + ) + ss[4], ss[0] = RND( + ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 47, 0x106AA07032BBD1B8 + ) + ss[3], ss[7] = RND( + ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 48, 0x19A4C116B8D2D0C8 + ) + ss[2], ss[6] = RND( + ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 49, 0x1E376C085141AB53 + ) + ss[1], ss[5] = RND( + ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 50, 0x2748774CDF8EEB99 + ) + ss[0], ss[4] = RND( + ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 51, 0x34B0BCB5E19B48A8 + ) + ss[7], ss[3] = RND( + ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 52, 0x391C0CB3C5C95A63 + ) + ss[6], ss[2] = RND( + ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 53, 0x4ED8AA4AE3418ACB + ) + ss[5], ss[1] = RND( + ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 54, 0x5B9CCA4F7763E373 + ) + ss[4], ss[0] = RND( + ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 55, 0x682E6FF3D6B2B8A3 + ) + ss[3], ss[7] = RND( + ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 56, 0x748F82EE5DEFB2FC + ) + ss[2], ss[6] = RND( + ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 57, 0x78A5636F43172F60 + ) + ss[1], ss[5] = RND( + ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 58, 0x84C87814A1F0AB72 + ) + ss[0], ss[4] = RND( + ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 59, 0x8CC702081A6439EC + ) + ss[7], ss[3] = RND( + ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 60, 0x90BEFFFA23631E28 + ) + ss[6], ss[2] = RND( + ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 61, 0xA4506CEBDE82BDE9 + ) + ss[5], ss[1] = RND( + ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 62, 0xBEF9A3F7B2C67915 + ) + ss[4], ss[0] = RND( + ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 63, 0xC67178F2E372532B + ) + ss[3], ss[7] = RND( + ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 64, 0xCA273ECEEA26619C + ) + ss[2], ss[6] = RND( + ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 65, 0xD186B8C721C0C207 + ) + ss[1], ss[5] = RND( + ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 66, 0xEADA7DD6CDE0EB1E + ) + ss[0], ss[4] = RND( + ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 67, 0xF57D4F7FEE6ED178 + ) + ss[7], ss[3] = RND( + ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 68, 0x06F067AA72176FBA + ) + ss[6], ss[2] = RND( + ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 69, 0x0A637DC5A2C898A6 + ) + ss[5], ss[1] = RND( + ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 70, 0x113F9804BEF90DAE + ) + ss[4], ss[0] = RND( + ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 71, 0x1B710B35131C471B + ) + ss[3], ss[7] = RND( + ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 72, 0x28DB77F523047D84 + ) + ss[2], ss[6] = RND( + ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 73, 0x32CAAB7B40C72493 + ) + ss[1], ss[5] = RND( + ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 74, 0x3C9EBE0A15C9BEBC + ) + ss[0], ss[4] = RND( + ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 75, 0x431D67C49C100D4C + ) + ss[7], ss[3] = RND( + ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 76, 0x4CC5D4BECB3E42B6 + ) + ss[6], ss[2] = RND( + ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 77, 0x597F299CFC657E2A + ) + ss[5], ss[1] = RND( + ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 78, 0x5FCB6FAB3AD6FAEC + ) + ss[4], ss[0] = RND( + ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 79, 0x6C44198C4A475817 + ) + + for i in range(len(self._digest)): + self._digest[i] = (self._digest[i] + ss[i]) & 0xFFFFFFFFFFFFFFFF + + def _update(self, buffer): + if isinstance(buffer, str): + raise TypeError("Unicode strings must be encoded before hashing") + count = len(buffer) + buffer_idx = 0 + clo = (self._count_lo + (count << 3)) & 0xFFFFFFFF + if clo < self._count_lo: + self._count_hi += 1 + self._count_lo = clo + + self._count_hi += count >> 29 + + if self._local: + i = _SHA_BLOCKSIZE - self._local + if i > count: + i = count + + # copy buffer + for x in enumerate(buffer[buffer_idx : buffer_idx + i]): + self._data[self._local + x[0]] = x[1] + + count -= i + buffer_idx += i + + self._local += i + if self._local == _SHA_BLOCKSIZE: + self._transform() + self._local = 0 + else: + return + + while count >= _SHA_BLOCKSIZE: + # copy buffer + self._data = bytearray(buffer[buffer_idx : buffer_idx + _SHA_BLOCKSIZE]) + count -= _SHA_BLOCKSIZE + buffer_idx += _SHA_BLOCKSIZE + self._transform() + + # copy buffer + pos = self._local + self._data[pos : pos + count] = buffer[buffer_idx : buffer_idx + count] + self._local = count + + def _final(self): + lo_bit_count = self._count_lo + hi_bit_count = self._count_hi + count = (lo_bit_count >> 3) & 0x7F + self._data[count] = 0x80 + count += 1 + if count > _SHA_BLOCKSIZE - 16: + # zero the bytes in data after the count + self._data = self._data[:count] + bytes(_SHA_BLOCKSIZE - count) + self._transform() + # zero bytes in data + self._data = bytearray(_SHA_BLOCKSIZE) + else: + self._data = self._data[:count] + bytes(_SHA_BLOCKSIZE - count) + + self._data[112] = 0 + self._data[113] = 0 + self._data[114] = 0 + self._data[115] = 0 + self._data[116] = 0 + self._data[117] = 0 + self._data[118] = 0 + self._data[119] = 0 + + self._data[120] = (hi_bit_count >> 24) & 0xFF + self._data[121] = (hi_bit_count >> 16) & 0xFF + self._data[122] = (hi_bit_count >> 8) & 0xFF + self._data[123] = (hi_bit_count >> 0) & 0xFF + self._data[124] = (lo_bit_count >> 24) & 0xFF + self._data[125] = (lo_bit_count >> 16) & 0xFF + self._data[126] = (lo_bit_count >> 8) & 0xFF + self._data[127] = (lo_bit_count >> 0) & 0xFF + + self._transform() + + dig = bytearray() + for i in self._digest: + for j in range(8): + dig.append((i >> ((7 - j) * 8)) & 0xFF) + return dig diff --git a/python-stdlib/hashlib-sha512/manifest.py b/python-stdlib/hashlib-sha512/manifest.py new file mode 100644 index 000000000..1d84f025a --- /dev/null +++ b/python-stdlib/hashlib-sha512/manifest.py @@ -0,0 +1,4 @@ +metadata(version="1.0.0", description="Adds the SHA512 hash algorithm to hashlib.") + +require("hashlib-core") +package("hashlib") diff --git a/python-stdlib/hashlib/hashlib/__init__.py b/python-stdlib/hashlib/hashlib/__init__.py deleted file mode 100644 index d7afbf819..000000000 --- a/python-stdlib/hashlib/hashlib/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -try: - import uhashlib -except ImportError: - uhashlib = None - - -def init(): - for i in ("sha1", "sha224", "sha256", "sha384", "sha512"): - c = getattr(uhashlib, i, None) - if not c: - c = __import__("_" + i, None, None, (), 1) - c = getattr(c, i) - globals()[i] = c - - -init() - - -def new(algo, data=b""): - try: - c = globals()[algo] - return c(data) - except KeyError: - raise ValueError(algo) diff --git a/python-stdlib/hashlib/hashlib/_sha224.py b/python-stdlib/hashlib/hashlib/_sha224.py deleted file mode 100644 index 634343b50..000000000 --- a/python-stdlib/hashlib/hashlib/_sha224.py +++ /dev/null @@ -1 +0,0 @@ -from ._sha256 import sha224 diff --git a/python-stdlib/hashlib/hashlib/_sha256.py b/python-stdlib/hashlib/hashlib/_sha256.py deleted file mode 100644 index e4bdeca4e..000000000 --- a/python-stdlib/hashlib/hashlib/_sha256.py +++ /dev/null @@ -1,301 +0,0 @@ -SHA_BLOCKSIZE = 64 -SHA_DIGESTSIZE = 32 - - -def new_shaobject(): - return { - "digest": [0] * 8, - "count_lo": 0, - "count_hi": 0, - "data": [0] * SHA_BLOCKSIZE, - "local": 0, - "digestsize": 0, - } - - -ROR = lambda x, y: (((x & 0xFFFFFFFF) >> (y & 31)) | (x << (32 - (y & 31)))) & 0xFFFFFFFF -Ch = lambda x, y, z: (z ^ (x & (y ^ z))) -Maj = lambda x, y, z: (((x | y) & z) | (x & y)) -S = lambda x, n: ROR(x, n) -R = lambda x, n: (x & 0xFFFFFFFF) >> n -Sigma0 = lambda x: (S(x, 2) ^ S(x, 13) ^ S(x, 22)) -Sigma1 = lambda x: (S(x, 6) ^ S(x, 11) ^ S(x, 25)) -Gamma0 = lambda x: (S(x, 7) ^ S(x, 18) ^ R(x, 3)) -Gamma1 = lambda x: (S(x, 17) ^ S(x, 19) ^ R(x, 10)) - - -def sha_transform(sha_info): - W = [] - - d = sha_info["data"] - for i in range(0, 16): - W.append((d[4 * i] << 24) + (d[4 * i + 1] << 16) + (d[4 * i + 2] << 8) + d[4 * i + 3]) - - for i in range(16, 64): - W.append((Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xFFFFFFFF) - - ss = sha_info["digest"][:] - - def RND(a, b, c, d, e, f, g, h, i, ki): - t0 = h + Sigma1(e) + Ch(e, f, g) + ki + W[i] - t1 = Sigma0(a) + Maj(a, b, c) - d += t0 - h = t0 + t1 - return d & 0xFFFFFFFF, h & 0xFFFFFFFF - - ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 0, 0x428A2F98) - ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 1, 0x71374491) - ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 2, 0xB5C0FBCF) - ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 3, 0xE9B5DBA5) - ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 4, 0x3956C25B) - ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 5, 0x59F111F1) - ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 6, 0x923F82A4) - ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 7, 0xAB1C5ED5) - ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 8, 0xD807AA98) - ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 9, 0x12835B01) - ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 10, 0x243185BE) - ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 11, 0x550C7DC3) - ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 12, 0x72BE5D74) - ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 13, 0x80DEB1FE) - ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 14, 0x9BDC06A7) - ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 15, 0xC19BF174) - ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 16, 0xE49B69C1) - ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 17, 0xEFBE4786) - ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 18, 0x0FC19DC6) - ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 19, 0x240CA1CC) - ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 20, 0x2DE92C6F) - ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 21, 0x4A7484AA) - ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 22, 0x5CB0A9DC) - ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 23, 0x76F988DA) - ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 24, 0x983E5152) - ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 25, 0xA831C66D) - ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 26, 0xB00327C8) - ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 27, 0xBF597FC7) - ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 28, 0xC6E00BF3) - ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 29, 0xD5A79147) - ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 30, 0x06CA6351) - ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 31, 0x14292967) - ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 32, 0x27B70A85) - ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 33, 0x2E1B2138) - ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 34, 0x4D2C6DFC) - ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 35, 0x53380D13) - ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 36, 0x650A7354) - ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 37, 0x766A0ABB) - ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 38, 0x81C2C92E) - ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 39, 0x92722C85) - ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 40, 0xA2BFE8A1) - ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 41, 0xA81A664B) - ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 42, 0xC24B8B70) - ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 43, 0xC76C51A3) - ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 44, 0xD192E819) - ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 45, 0xD6990624) - ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 46, 0xF40E3585) - ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 47, 0x106AA070) - ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 48, 0x19A4C116) - ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 49, 0x1E376C08) - ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 50, 0x2748774C) - ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 51, 0x34B0BCB5) - ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 52, 0x391C0CB3) - ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 53, 0x4ED8AA4A) - ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 54, 0x5B9CCA4F) - ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 55, 0x682E6FF3) - ss[3], ss[7] = RND(ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 56, 0x748F82EE) - ss[2], ss[6] = RND(ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 57, 0x78A5636F) - ss[1], ss[5] = RND(ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 58, 0x84C87814) - ss[0], ss[4] = RND(ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 59, 0x8CC70208) - ss[7], ss[3] = RND(ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 60, 0x90BEFFFA) - ss[6], ss[2] = RND(ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 61, 0xA4506CEB) - ss[5], ss[1] = RND(ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 62, 0xBEF9A3F7) - ss[4], ss[0] = RND(ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 63, 0xC67178F2) - - dig = [] - for i, x in enumerate(sha_info["digest"]): - dig.append((x + ss[i]) & 0xFFFFFFFF) - sha_info["digest"] = dig - - -def sha_init(): - sha_info = new_shaobject() - sha_info["digest"] = [ - 0x6A09E667, - 0xBB67AE85, - 0x3C6EF372, - 0xA54FF53A, - 0x510E527F, - 0x9B05688C, - 0x1F83D9AB, - 0x5BE0CD19, - ] - sha_info["count_lo"] = 0 - sha_info["count_hi"] = 0 - sha_info["local"] = 0 - sha_info["digestsize"] = 32 - return sha_info - - -def sha224_init(): - sha_info = new_shaobject() - sha_info["digest"] = [ - 0xC1059ED8, - 0x367CD507, - 0x3070DD17, - 0xF70E5939, - 0xFFC00B31, - 0x68581511, - 0x64F98FA7, - 0xBEFA4FA4, - ] - sha_info["count_lo"] = 0 - sha_info["count_hi"] = 0 - sha_info["local"] = 0 - sha_info["digestsize"] = 28 - return sha_info - - -def getbuf(s): - if isinstance(s, str): - return s.encode("ascii") - else: - return bytes(s) - - -def sha_update(sha_info, buffer): - if isinstance(buffer, str): - raise TypeError("Unicode strings must be encoded before hashing") - count = len(buffer) - buffer_idx = 0 - clo = (sha_info["count_lo"] + (count << 3)) & 0xFFFFFFFF - if clo < sha_info["count_lo"]: - sha_info["count_hi"] += 1 - sha_info["count_lo"] = clo - - sha_info["count_hi"] += count >> 29 - - if sha_info["local"]: - i = SHA_BLOCKSIZE - sha_info["local"] - if i > count: - i = count - - # copy buffer - for x in enumerate(buffer[buffer_idx : buffer_idx + i]): - sha_info["data"][sha_info["local"] + x[0]] = x[1] - - count -= i - buffer_idx += i - - sha_info["local"] += i - if sha_info["local"] == SHA_BLOCKSIZE: - sha_transform(sha_info) - sha_info["local"] = 0 - else: - return - - while count >= SHA_BLOCKSIZE: - # copy buffer - sha_info["data"] = list(buffer[buffer_idx : buffer_idx + SHA_BLOCKSIZE]) - count -= SHA_BLOCKSIZE - buffer_idx += SHA_BLOCKSIZE - sha_transform(sha_info) - - # copy buffer - pos = sha_info["local"] - sha_info["data"][pos : pos + count] = list(buffer[buffer_idx : buffer_idx + count]) - sha_info["local"] = count - - -def sha_final(sha_info): - lo_bit_count = sha_info["count_lo"] - hi_bit_count = sha_info["count_hi"] - count = (lo_bit_count >> 3) & 0x3F - sha_info["data"][count] = 0x80 - count += 1 - if count > SHA_BLOCKSIZE - 8: - # zero the bytes in data after the count - sha_info["data"] = sha_info["data"][:count] + ([0] * (SHA_BLOCKSIZE - count)) - sha_transform(sha_info) - # zero bytes in data - sha_info["data"] = [0] * SHA_BLOCKSIZE - else: - sha_info["data"] = sha_info["data"][:count] + ([0] * (SHA_BLOCKSIZE - count)) - - sha_info["data"][56] = (hi_bit_count >> 24) & 0xFF - sha_info["data"][57] = (hi_bit_count >> 16) & 0xFF - sha_info["data"][58] = (hi_bit_count >> 8) & 0xFF - sha_info["data"][59] = (hi_bit_count >> 0) & 0xFF - sha_info["data"][60] = (lo_bit_count >> 24) & 0xFF - sha_info["data"][61] = (lo_bit_count >> 16) & 0xFF - sha_info["data"][62] = (lo_bit_count >> 8) & 0xFF - sha_info["data"][63] = (lo_bit_count >> 0) & 0xFF - - sha_transform(sha_info) - - dig = [] - for i in sha_info["digest"]: - dig.extend([((i >> 24) & 0xFF), ((i >> 16) & 0xFF), ((i >> 8) & 0xFF), (i & 0xFF)]) - return bytes(dig) - - -class sha256(object): - digest_size = digestsize = SHA_DIGESTSIZE - block_size = SHA_BLOCKSIZE - - def __init__(self, s=None): - self._sha = sha_init() - if s: - sha_update(self._sha, getbuf(s)) - - def update(self, s): - sha_update(self._sha, getbuf(s)) - - def digest(self): - return sha_final(self._sha.copy())[: self._sha["digestsize"]] - - def hexdigest(self): - return "".join(["%.2x" % i for i in self.digest()]) - - def copy(self): - new = sha256() - new._sha = self._sha.copy() - return new - - -class sha224(sha256): - digest_size = digestsize = 28 - - def __init__(self, s=None): - self._sha = sha224_init() - if s: - sha_update(self._sha, getbuf(s)) - - def copy(self): - new = sha224() - new._sha = self._sha.copy() - return new - - -def test(): - a_str = "just a test string" - - assert ( - b"\xe3\xb0\xc4B\x98\xfc\x1c\x14\x9a\xfb\xf4\xc8\x99o\xb9$'\xaeA\xe4d\x9b\x93L\xa4\x95\x99\x1bxR\xb8U" - == sha256().digest() - ) - assert ( - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" == sha256().hexdigest() - ) - assert ( - "d7b553c6f09ac85d142415f857c5310f3bbbe7cdd787cce4b985acedd585266f" - == sha256(a_str).hexdigest() - ) - assert ( - "8113ebf33c97daa9998762aacafe750c7cefc2b2f173c90c59663a57fe626f21" - == sha256(a_str * 7).hexdigest() - ) - - s = sha256(a_str) - s.update(a_str) - assert "03d9963e05a094593190b6fc794cb1a3e1ac7d7883f0b5855268afeccc70d461" == s.hexdigest() - - -if __name__ == "__main__": - test() diff --git a/python-stdlib/hashlib/hashlib/_sha384.py b/python-stdlib/hashlib/hashlib/_sha384.py deleted file mode 100644 index 20f09ff00..000000000 --- a/python-stdlib/hashlib/hashlib/_sha384.py +++ /dev/null @@ -1 +0,0 @@ -from ._sha512 import sha384 diff --git a/python-stdlib/hashlib/hashlib/_sha512.py b/python-stdlib/hashlib/hashlib/_sha512.py deleted file mode 100644 index 726fbb5f2..000000000 --- a/python-stdlib/hashlib/hashlib/_sha512.py +++ /dev/null @@ -1,519 +0,0 @@ -""" -This code was Ported from CPython's sha512module.c -""" - -SHA_BLOCKSIZE = 128 -SHA_DIGESTSIZE = 64 - - -def new_shaobject(): - return { - "digest": [0] * 8, - "count_lo": 0, - "count_hi": 0, - "data": [0] * SHA_BLOCKSIZE, - "local": 0, - "digestsize": 0, - } - - -ROR64 = ( - lambda x, y: (((x & 0xFFFFFFFFFFFFFFFF) >> (y & 63)) | (x << (64 - (y & 63)))) - & 0xFFFFFFFFFFFFFFFF -) -Ch = lambda x, y, z: (z ^ (x & (y ^ z))) -Maj = lambda x, y, z: (((x | y) & z) | (x & y)) -S = lambda x, n: ROR64(x, n) -R = lambda x, n: (x & 0xFFFFFFFFFFFFFFFF) >> n -Sigma0 = lambda x: (S(x, 28) ^ S(x, 34) ^ S(x, 39)) -Sigma1 = lambda x: (S(x, 14) ^ S(x, 18) ^ S(x, 41)) -Gamma0 = lambda x: (S(x, 1) ^ S(x, 8) ^ R(x, 7)) -Gamma1 = lambda x: (S(x, 19) ^ S(x, 61) ^ R(x, 6)) - - -def sha_transform(sha_info): - W = [] - - d = sha_info["data"] - for i in range(0, 16): - W.append( - (d[8 * i] << 56) - + (d[8 * i + 1] << 48) - + (d[8 * i + 2] << 40) - + (d[8 * i + 3] << 32) - + (d[8 * i + 4] << 24) - + (d[8 * i + 5] << 16) - + (d[8 * i + 6] << 8) - + d[8 * i + 7] - ) - - for i in range(16, 80): - W.append( - (Gamma1(W[i - 2]) + W[i - 7] + Gamma0(W[i - 15]) + W[i - 16]) & 0xFFFFFFFFFFFFFFFF - ) - - ss = sha_info["digest"][:] - - def RND(a, b, c, d, e, f, g, h, i, ki): - t0 = (h + Sigma1(e) + Ch(e, f, g) + ki + W[i]) & 0xFFFFFFFFFFFFFFFF - t1 = (Sigma0(a) + Maj(a, b, c)) & 0xFFFFFFFFFFFFFFFF - d = (d + t0) & 0xFFFFFFFFFFFFFFFF - h = (t0 + t1) & 0xFFFFFFFFFFFFFFFF - return d & 0xFFFFFFFFFFFFFFFF, h & 0xFFFFFFFFFFFFFFFF - - ss[3], ss[7] = RND( - ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 0, 0x428A2F98D728AE22 - ) - ss[2], ss[6] = RND( - ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 1, 0x7137449123EF65CD - ) - ss[1], ss[5] = RND( - ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 2, 0xB5C0FBCFEC4D3B2F - ) - ss[0], ss[4] = RND( - ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 3, 0xE9B5DBA58189DBBC - ) - ss[7], ss[3] = RND( - ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 4, 0x3956C25BF348B538 - ) - ss[6], ss[2] = RND( - ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 5, 0x59F111F1B605D019 - ) - ss[5], ss[1] = RND( - ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 6, 0x923F82A4AF194F9B - ) - ss[4], ss[0] = RND( - ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 7, 0xAB1C5ED5DA6D8118 - ) - ss[3], ss[7] = RND( - ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 8, 0xD807AA98A3030242 - ) - ss[2], ss[6] = RND( - ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 9, 0x12835B0145706FBE - ) - ss[1], ss[5] = RND( - ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 10, 0x243185BE4EE4B28C - ) - ss[0], ss[4] = RND( - ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 11, 0x550C7DC3D5FFB4E2 - ) - ss[7], ss[3] = RND( - ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 12, 0x72BE5D74F27B896F - ) - ss[6], ss[2] = RND( - ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 13, 0x80DEB1FE3B1696B1 - ) - ss[5], ss[1] = RND( - ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 14, 0x9BDC06A725C71235 - ) - ss[4], ss[0] = RND( - ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 15, 0xC19BF174CF692694 - ) - ss[3], ss[7] = RND( - ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 16, 0xE49B69C19EF14AD2 - ) - ss[2], ss[6] = RND( - ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 17, 0xEFBE4786384F25E3 - ) - ss[1], ss[5] = RND( - ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 18, 0x0FC19DC68B8CD5B5 - ) - ss[0], ss[4] = RND( - ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 19, 0x240CA1CC77AC9C65 - ) - ss[7], ss[3] = RND( - ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 20, 0x2DE92C6F592B0275 - ) - ss[6], ss[2] = RND( - ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 21, 0x4A7484AA6EA6E483 - ) - ss[5], ss[1] = RND( - ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 22, 0x5CB0A9DCBD41FBD4 - ) - ss[4], ss[0] = RND( - ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 23, 0x76F988DA831153B5 - ) - ss[3], ss[7] = RND( - ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 24, 0x983E5152EE66DFAB - ) - ss[2], ss[6] = RND( - ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 25, 0xA831C66D2DB43210 - ) - ss[1], ss[5] = RND( - ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 26, 0xB00327C898FB213F - ) - ss[0], ss[4] = RND( - ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 27, 0xBF597FC7BEEF0EE4 - ) - ss[7], ss[3] = RND( - ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 28, 0xC6E00BF33DA88FC2 - ) - ss[6], ss[2] = RND( - ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 29, 0xD5A79147930AA725 - ) - ss[5], ss[1] = RND( - ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 30, 0x06CA6351E003826F - ) - ss[4], ss[0] = RND( - ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 31, 0x142929670A0E6E70 - ) - ss[3], ss[7] = RND( - ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 32, 0x27B70A8546D22FFC - ) - ss[2], ss[6] = RND( - ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 33, 0x2E1B21385C26C926 - ) - ss[1], ss[5] = RND( - ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 34, 0x4D2C6DFC5AC42AED - ) - ss[0], ss[4] = RND( - ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 35, 0x53380D139D95B3DF - ) - ss[7], ss[3] = RND( - ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 36, 0x650A73548BAF63DE - ) - ss[6], ss[2] = RND( - ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 37, 0x766A0ABB3C77B2A8 - ) - ss[5], ss[1] = RND( - ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 38, 0x81C2C92E47EDAEE6 - ) - ss[4], ss[0] = RND( - ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 39, 0x92722C851482353B - ) - ss[3], ss[7] = RND( - ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 40, 0xA2BFE8A14CF10364 - ) - ss[2], ss[6] = RND( - ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 41, 0xA81A664BBC423001 - ) - ss[1], ss[5] = RND( - ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 42, 0xC24B8B70D0F89791 - ) - ss[0], ss[4] = RND( - ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 43, 0xC76C51A30654BE30 - ) - ss[7], ss[3] = RND( - ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 44, 0xD192E819D6EF5218 - ) - ss[6], ss[2] = RND( - ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 45, 0xD69906245565A910 - ) - ss[5], ss[1] = RND( - ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 46, 0xF40E35855771202A - ) - ss[4], ss[0] = RND( - ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 47, 0x106AA07032BBD1B8 - ) - ss[3], ss[7] = RND( - ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 48, 0x19A4C116B8D2D0C8 - ) - ss[2], ss[6] = RND( - ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 49, 0x1E376C085141AB53 - ) - ss[1], ss[5] = RND( - ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 50, 0x2748774CDF8EEB99 - ) - ss[0], ss[4] = RND( - ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 51, 0x34B0BCB5E19B48A8 - ) - ss[7], ss[3] = RND( - ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 52, 0x391C0CB3C5C95A63 - ) - ss[6], ss[2] = RND( - ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 53, 0x4ED8AA4AE3418ACB - ) - ss[5], ss[1] = RND( - ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 54, 0x5B9CCA4F7763E373 - ) - ss[4], ss[0] = RND( - ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 55, 0x682E6FF3D6B2B8A3 - ) - ss[3], ss[7] = RND( - ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 56, 0x748F82EE5DEFB2FC - ) - ss[2], ss[6] = RND( - ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 57, 0x78A5636F43172F60 - ) - ss[1], ss[5] = RND( - ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 58, 0x84C87814A1F0AB72 - ) - ss[0], ss[4] = RND( - ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 59, 0x8CC702081A6439EC - ) - ss[7], ss[3] = RND( - ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 60, 0x90BEFFFA23631E28 - ) - ss[6], ss[2] = RND( - ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 61, 0xA4506CEBDE82BDE9 - ) - ss[5], ss[1] = RND( - ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 62, 0xBEF9A3F7B2C67915 - ) - ss[4], ss[0] = RND( - ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 63, 0xC67178F2E372532B - ) - ss[3], ss[7] = RND( - ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 64, 0xCA273ECEEA26619C - ) - ss[2], ss[6] = RND( - ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 65, 0xD186B8C721C0C207 - ) - ss[1], ss[5] = RND( - ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 66, 0xEADA7DD6CDE0EB1E - ) - ss[0], ss[4] = RND( - ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 67, 0xF57D4F7FEE6ED178 - ) - ss[7], ss[3] = RND( - ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 68, 0x06F067AA72176FBA - ) - ss[6], ss[2] = RND( - ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 69, 0x0A637DC5A2C898A6 - ) - ss[5], ss[1] = RND( - ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 70, 0x113F9804BEF90DAE - ) - ss[4], ss[0] = RND( - ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 71, 0x1B710B35131C471B - ) - ss[3], ss[7] = RND( - ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], 72, 0x28DB77F523047D84 - ) - ss[2], ss[6] = RND( - ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], 73, 0x32CAAB7B40C72493 - ) - ss[1], ss[5] = RND( - ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], ss[5], 74, 0x3C9EBE0A15C9BEBC - ) - ss[0], ss[4] = RND( - ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], ss[4], 75, 0x431D67C49C100D4C - ) - ss[7], ss[3] = RND( - ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], ss[3], 76, 0x4CC5D4BECB3E42B6 - ) - ss[6], ss[2] = RND( - ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], ss[2], 77, 0x597F299CFC657E2A - ) - ss[5], ss[1] = RND( - ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], ss[1], 78, 0x5FCB6FAB3AD6FAEC - ) - ss[4], ss[0] = RND( - ss[1], ss[2], ss[3], ss[4], ss[5], ss[6], ss[7], ss[0], 79, 0x6C44198C4A475817 - ) - - dig = [] - for i, x in enumerate(sha_info["digest"]): - dig.append((x + ss[i]) & 0xFFFFFFFFFFFFFFFF) - sha_info["digest"] = dig - - -def sha_init(): - sha_info = new_shaobject() - sha_info["digest"] = [ - 0x6A09E667F3BCC908, - 0xBB67AE8584CAA73B, - 0x3C6EF372FE94F82B, - 0xA54FF53A5F1D36F1, - 0x510E527FADE682D1, - 0x9B05688C2B3E6C1F, - 0x1F83D9ABFB41BD6B, - 0x5BE0CD19137E2179, - ] - sha_info["count_lo"] = 0 - sha_info["count_hi"] = 0 - sha_info["local"] = 0 - sha_info["digestsize"] = 64 - return sha_info - - -def sha384_init(): - sha_info = new_shaobject() - sha_info["digest"] = [ - 0xCBBB9D5DC1059ED8, - 0x629A292A367CD507, - 0x9159015A3070DD17, - 0x152FECD8F70E5939, - 0x67332667FFC00B31, - 0x8EB44A8768581511, - 0xDB0C2E0D64F98FA7, - 0x47B5481DBEFA4FA4, - ] - sha_info["count_lo"] = 0 - sha_info["count_hi"] = 0 - sha_info["local"] = 0 - sha_info["digestsize"] = 48 - return sha_info - - -def getbuf(s): - if isinstance(s, str): - return s.encode("ascii") - else: - return bytes(s) - - -def sha_update(sha_info, buffer): - if isinstance(buffer, str): - raise TypeError("Unicode strings must be encoded before hashing") - count = len(buffer) - buffer_idx = 0 - clo = (sha_info["count_lo"] + (count << 3)) & 0xFFFFFFFF - if clo < sha_info["count_lo"]: - sha_info["count_hi"] += 1 - sha_info["count_lo"] = clo - - sha_info["count_hi"] += count >> 29 - - if sha_info["local"]: - i = SHA_BLOCKSIZE - sha_info["local"] - if i > count: - i = count - - # copy buffer - for x in enumerate(buffer[buffer_idx : buffer_idx + i]): - sha_info["data"][sha_info["local"] + x[0]] = x[1] - - count -= i - buffer_idx += i - - sha_info["local"] += i - if sha_info["local"] == SHA_BLOCKSIZE: - sha_transform(sha_info) - sha_info["local"] = 0 - else: - return - - while count >= SHA_BLOCKSIZE: - # copy buffer - sha_info["data"] = list(buffer[buffer_idx : buffer_idx + SHA_BLOCKSIZE]) - count -= SHA_BLOCKSIZE - buffer_idx += SHA_BLOCKSIZE - sha_transform(sha_info) - - # copy buffer - pos = sha_info["local"] - sha_info["data"][pos : pos + count] = list(buffer[buffer_idx : buffer_idx + count]) - sha_info["local"] = count - - -def sha_final(sha_info): - lo_bit_count = sha_info["count_lo"] - hi_bit_count = sha_info["count_hi"] - count = (lo_bit_count >> 3) & 0x7F - sha_info["data"][count] = 0x80 - count += 1 - if count > SHA_BLOCKSIZE - 16: - # zero the bytes in data after the count - sha_info["data"] = sha_info["data"][:count] + ([0] * (SHA_BLOCKSIZE - count)) - sha_transform(sha_info) - # zero bytes in data - sha_info["data"] = [0] * SHA_BLOCKSIZE - else: - sha_info["data"] = sha_info["data"][:count] + ([0] * (SHA_BLOCKSIZE - count)) - - sha_info["data"][112] = 0 - sha_info["data"][113] = 0 - sha_info["data"][114] = 0 - sha_info["data"][115] = 0 - sha_info["data"][116] = 0 - sha_info["data"][117] = 0 - sha_info["data"][118] = 0 - sha_info["data"][119] = 0 - - sha_info["data"][120] = (hi_bit_count >> 24) & 0xFF - sha_info["data"][121] = (hi_bit_count >> 16) & 0xFF - sha_info["data"][122] = (hi_bit_count >> 8) & 0xFF - sha_info["data"][123] = (hi_bit_count >> 0) & 0xFF - sha_info["data"][124] = (lo_bit_count >> 24) & 0xFF - sha_info["data"][125] = (lo_bit_count >> 16) & 0xFF - sha_info["data"][126] = (lo_bit_count >> 8) & 0xFF - sha_info["data"][127] = (lo_bit_count >> 0) & 0xFF - - sha_transform(sha_info) - - dig = [] - for i in sha_info["digest"]: - dig.extend( - [ - ((i >> 56) & 0xFF), - ((i >> 48) & 0xFF), - ((i >> 40) & 0xFF), - ((i >> 32) & 0xFF), - ((i >> 24) & 0xFF), - ((i >> 16) & 0xFF), - ((i >> 8) & 0xFF), - (i & 0xFF), - ] - ) - return bytes(dig) - - -class sha512(object): - digest_size = digestsize = SHA_DIGESTSIZE - block_size = SHA_BLOCKSIZE - - def __init__(self, s=None): - self._sha = sha_init() - if s: - sha_update(self._sha, getbuf(s)) - - def update(self, s): - sha_update(self._sha, getbuf(s)) - - def digest(self): - return sha_final(self._sha.copy())[: self._sha["digestsize"]] - - def hexdigest(self): - return "".join(["%.2x" % i for i in self.digest()]) - - def copy(self): - new = sha512() - new._sha = self._sha.copy() - return new - - -class sha384(sha512): - digest_size = digestsize = 48 - - def __init__(self, s=None): - self._sha = sha384_init() - if s: - sha_update(self._sha, getbuf(s)) - - def copy(self): - new = sha384() - new._sha = self._sha.copy() - return new - - -def test(): - a_str = "just a test string" - - assert ( - sha512().digest() - == b"\xcf\x83\xe15~\xef\xb8\xbd\xf1T(P\xd6m\x80\x07\xd6 \xe4\x05\x0bW\x15\xdc\x83\xf4\xa9!\xd3l\xe9\xceG\xd0\xd1<]\x85\xf2\xb0\xff\x83\x18\xd2\x87~\xec/c\xb91\xbdGAz\x81\xa582z\xf9'\xda>" - ) - assert ( - sha512().hexdigest() - == "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" - ) - assert ( - sha512(a_str).hexdigest() - == "68be4c6664af867dd1d01c8d77e963d87d77b702400c8fabae355a41b8927a5a5533a7f1c28509bbd65c5f3ac716f33be271fbda0ca018b71a84708c9fae8a53" - ) - assert ( - sha512(a_str * 7).hexdigest() - == "3233acdbfcfff9bff9fc72401d31dbffa62bd24e9ec846f0578d647da73258d9f0879f7fde01fe2cc6516af3f343807fdef79e23d696c923d79931db46bf1819" - ) - - s = sha512(a_str) - s.update(a_str) - assert ( - s.hexdigest() - == "341aeb668730bbb48127d5531115f3c39d12cb9586a6ca770898398aff2411087cfe0b570689adf328cddeb1f00803acce6737a19f310b53bbdb0320828f75bb" - ) - - -if __name__ == "__main__": - test() diff --git a/python-stdlib/hashlib/manifest.py b/python-stdlib/hashlib/manifest.py index 106168bab..01f745e14 100644 --- a/python-stdlib/hashlib/manifest.py +++ b/python-stdlib/hashlib/manifest.py @@ -1,3 +1,8 @@ -metadata(version="2.4.0-4") +metadata(version="2.5.0") -package("hashlib") +# This is a collection package that gets all hash functions. To save code and +# memory size, prefer to install just the algorithm you need. +require("hashlib-sha224") +require("hashlib-sha256") +require("hashlib-sha384") +require("hashlib-sha512") diff --git a/python-stdlib/hashlib/test_hashlib.py b/python-stdlib/hashlib/test_hashlib.py deleted file mode 100644 index 6cb687399..000000000 --- a/python-stdlib/hashlib/test_hashlib.py +++ /dev/null @@ -1,35 +0,0 @@ -from hashlib._sha256 import test as sha256_test -from hashlib._sha512 import test as sha512_test - - -sha256_test() -sha512_test() - - -import hashlib - -patterns = [ - ( - "sha224", - b"1234", - b"\x99\xfb/H\xc6\xafGa\xf9\x04\xfc\x85\xf9^\xb5a\x90\xe5\xd4\x0b\x1fD\xec:\x9c\x1f\xa3\x19", - ), - ( - "sha256", - b"1234", - b"\x03\xacgB\x16\xf3\xe1\\v\x1e\xe1\xa5\xe2U\xf0g\x956#\xc8\xb3\x88\xb4E\x9e\x13\xf9x\xd7\xc8F\xf4", - ), - ( - "sha384", - b"1234", - b"PO\x00\x8c\x8f\xcf\x8b.\xd5\xdf\xcd\xe7R\xfcTd\xab\x8b\xa0d!]\x9c[_\xc4\x86\xaf=\x9a\xb8\xc8\x1b\x14xQ\x80\xd2\xad|\xee\x1a\xb7\x92\xadDy\x8c", - ), - ( - "sha512", - b"1234", - b"\xd4\x04U\x9f`.\xabo\xd6\x02\xacv\x80\xda\xcb\xfa\xad\xd1603^\x95\x1f\tz\xf3\x90\x0e\x9d\xe1v\xb6\xdb(Q/.\x00\x0b\x9d\x04\xfb\xa5\x13>\x8b\x1cn\x8d\xf5\x9d\xb3\xa8\xab\x9d`\xbeK\x97\xcc\x9e\x81\xdb", - ), -] - -for algo, input, output in patterns: - assert hashlib.new(algo, input).digest() == output diff --git a/python-stdlib/hashlib/tests/test_new.py b/python-stdlib/hashlib/tests/test_new.py new file mode 100644 index 000000000..f844a1ccd --- /dev/null +++ b/python-stdlib/hashlib/tests/test_new.py @@ -0,0 +1,32 @@ +import unittest +import hashlib + + +class TestNew(unittest.TestCase): + def test_sha224(self): + self.assertEqual( + hashlib.new("sha224", b"1234").digest(), + b"\x99\xfb/H\xc6\xafGa\xf9\x04\xfc\x85\xf9^\xb5a\x90\xe5\xd4\x0b\x1fD\xec:\x9c\x1f\xa3\x19", + ) + + def test_sha256(self): + self.assertEqual( + hashlib.new("sha256", b"1234").digest(), + b"\x03\xacgB\x16\xf3\xe1\\v\x1e\xe1\xa5\xe2U\xf0g\x956#\xc8\xb3\x88\xb4E\x9e\x13\xf9x\xd7\xc8F\xf4", + ) + + def test_sha384(self): + self.assertEqual( + hashlib.new("sha384", b"1234").digest(), + b"PO\x00\x8c\x8f\xcf\x8b.\xd5\xdf\xcd\xe7R\xfcTd\xab\x8b\xa0d!]\x9c[_\xc4\x86\xaf=\x9a\xb8\xc8\x1b\x14xQ\x80\xd2\xad|\xee\x1a\xb7\x92\xadDy\x8c", + ) + + def test_sha512(self): + self.assertEqual( + hashlib.new("sha512", b"1234").digest(), + b"\xd4\x04U\x9f`.\xabo\xd6\x02\xacv\x80\xda\xcb\xfa\xad\xd1603^\x95\x1f\tz\xf3\x90\x0e\x9d\xe1v\xb6\xdb(Q/.\x00\x0b\x9d\x04\xfb\xa5\x13>\x8b\x1cn\x8d\xf5\x9d\xb3\xa8\xab\x9d`\xbeK\x97\xcc\x9e\x81\xdb", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/python-stdlib/hashlib/tests/test_sha256.py b/python-stdlib/hashlib/tests/test_sha256.py new file mode 100644 index 000000000..a311a8cc9 --- /dev/null +++ b/python-stdlib/hashlib/tests/test_sha256.py @@ -0,0 +1,87 @@ +# Prevent importing any built-in hashes, so this test tests only the pure Python hashes. +import sys +sys.modules['uhashlib'] = sys + +import unittest +from hashlib import sha256 + + +class TestSha256(unittest.TestCase): + a_str = b"just a test string" + b_str = b"some other string for testing" + c_str = b"nothing to see here" + + def test_empty(self): + self.assertEqual( + b"\xe3\xb0\xc4B\x98\xfc\x1c\x14\x9a\xfb\xf4\xc8\x99o\xb9$'\xaeA\xe4d\x9b\x93L\xa4\x95\x99\x1bxR\xb8U", + sha256().digest(), + ) + + def test_empty_hex(self): + self.assertEqual( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + sha256().hexdigest(), + ) + + def test_str(self): + self.assertEqual( + b"\xd7\xb5S\xc6\xf0\x9a\xc8]\x14$\x15\xf8W\xc51\x0f;\xbb\xe7\xcd\xd7\x87\xcc\xe4\xb9\x85\xac\xed\xd5\x85&o", + sha256(self.a_str).digest(), + ) + self.assertEqual( + b'|\x80Q\xb2\xa0u\xf0g\xe3\xc45\xce1p\xc6I\xb6r\x19J&\x8b\xdc\xa5"\x00?A\x90\xba\xbd,', + sha256(self.b_str).digest(), + ) + + def test_str_hex(self): + self.assertEqual( + "d7b553c6f09ac85d142415f857c5310f3bbbe7cdd787cce4b985acedd585266f", + sha256(self.a_str).hexdigest(), + ) + self.assertEqual( + "7c8051b2a075f067e3c435ce3170c649b672194a268bdca522003f4190babd2c", + sha256(self.b_str).hexdigest(), + ) + + def test_long_str(self): + self.assertEqual( + "f1f1af5d66ba1789f8214354c0ed04856bbe43c01aa392c584ef1ec3dbf45482", + sha256(self.a_str * 123).hexdigest(), + ) + + def test_update(self): + s = sha256(self.a_str) + s.update(self.b_str) + self.assertEqual( + "fc7f204eb969ca3f10488731fa63910486adda7c2ae2ee2142e85414454c6d42", s.hexdigest() + ) + + def test_repeat_final(self): + s = sha256(self.a_str) + s.update(self.b_str) + self.assertEqual( + "fc7f204eb969ca3f10488731fa63910486adda7c2ae2ee2142e85414454c6d42", s.hexdigest() + ) + self.assertEqual( + "fc7f204eb969ca3f10488731fa63910486adda7c2ae2ee2142e85414454c6d42", s.hexdigest() + ) + s.update(self.c_str) + self.assertEqual( + "b707db9ae915b0f6f9a67ded8c9932999ee7e9dfb33513b084ea9384f5ffb082", s.hexdigest() + ) + + def test_copy(self): + s = sha256(self.a_str) + s2 = s.copy() + s.update(self.b_str) + s2.update(self.c_str) + self.assertEqual( + "fc7f204eb969ca3f10488731fa63910486adda7c2ae2ee2142e85414454c6d42", s.hexdigest() + ) + self.assertEqual( + "6a340b2bd2b63f4a0f9bb7566c26831354ee6ed17d1187d3a53627181fcb2907", s2.hexdigest() + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/python-stdlib/hashlib/tests/test_sha512.py b/python-stdlib/hashlib/tests/test_sha512.py new file mode 100644 index 000000000..9f80606f3 --- /dev/null +++ b/python-stdlib/hashlib/tests/test_sha512.py @@ -0,0 +1,89 @@ +import unittest +from hashlib import sha512 + + +class Testsha512(unittest.TestCase): + a_str = b"just a test string" + b_str = b"some other string for testing" + c_str = b"nothing to see here" + + def test_empty(self): + self.assertEqual( + b"\xcf\x83\xe15~\xef\xb8\xbd\xf1T(P\xd6m\x80\x07\xd6 \xe4\x05\x0bW\x15\xdc\x83\xf4\xa9!\xd3l\xe9\xceG\xd0\xd1<]\x85\xf2\xb0\xff\x83\x18\xd2\x87~\xec/c\xb91\xbdGAz\x81\xa582z\xf9'\xda>", + sha512().digest(), + ) + + def test_empty_hex(self): + self.assertEqual( + "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e", + sha512().hexdigest(), + ) + + def test_str(self): + self.assertEqual( + b"h\xbeLfd\xaf\x86}\xd1\xd0\x1c\x8dw\xe9c\xd8}w\xb7\x02@\x0c\x8f\xab\xae5ZA\xb8\x92zZU3\xa7\xf1\xc2\x85\t\xbb\xd6\\_:\xc7\x16\xf3;\xe2q\xfb\xda\x0c\xa0\x18\xb7\x1a\x84p\x8c\x9f\xae\x8aS", + sha512(self.a_str).digest(), + ) + self.assertEqual( + b"Tt\xd1\xf8\x1fh\x14\xba\x85\x1a\x84\x15\x9b(\x812\x8er\x8d\xdeN\xc0\xe2\xff\xbb\xcc$i\x18gh\x18\xc4\xcb?\xc0\xa0\nTl\x0f\x01J\x07eP\x19\x98\xd9\xebZ\xd2?\x1cj\xa8Q)!\x18\xab!!~", + sha512(self.b_str).digest(), + ) + + def test_str_hex(self): + self.assertEqual( + "68be4c6664af867dd1d01c8d77e963d87d77b702400c8fabae355a41b8927a5a5533a7f1c28509bbd65c5f3ac716f33be271fbda0ca018b71a84708c9fae8a53", + sha512(self.a_str).hexdigest(), + ) + self.assertEqual( + "5474d1f81f6814ba851a84159b2881328e728dde4ec0e2ffbbcc246918676818c4cb3fc0a00a546c0f014a0765501998d9eb5ad23f1c6aa851292118ab21217e", + sha512(self.b_str).hexdigest(), + ) + + def test_long_str(self): + self.assertEqual( + "8ee045cd8faf900bb23d13754d65723404a224030af827897cde92a40f7a1202405bc3efe5466c7e4833e7a9a5b9f9278ebe4c968e7fa662d8addc17ba95cc73", + sha512(self.a_str * 123).hexdigest(), + ) + + def test_update(self): + s = sha512(self.a_str) + s.update(self.b_str) + self.assertEqual( + "3fa253e7b093d5bc7b31f613f03833a4d39341cf73642349a46f26b39b5d95c97bb4e16fc588bda81d5c7a2db62cfca5c4c71a142cf02fd78409bffe5e4f408c", + s.hexdigest(), + ) + + def test_repeat_final(self): + s = sha512(self.a_str) + s.update(self.b_str) + self.assertEqual( + "3fa253e7b093d5bc7b31f613f03833a4d39341cf73642349a46f26b39b5d95c97bb4e16fc588bda81d5c7a2db62cfca5c4c71a142cf02fd78409bffe5e4f408c", + s.hexdigest(), + ) + self.assertEqual( + "3fa253e7b093d5bc7b31f613f03833a4d39341cf73642349a46f26b39b5d95c97bb4e16fc588bda81d5c7a2db62cfca5c4c71a142cf02fd78409bffe5e4f408c", + s.hexdigest(), + ) + s.update(self.c_str) + self.assertEqual( + "4b0827d5a28eeb2ebbeec270d7c775e78d5a76251753b8242327ffa2b1e5662a655be44bc09e41fcc0805bccd79cee13f4c41c40acff6fc1cf69b311d9b08f55", + s.hexdigest(), + ) + + def test_copy(self): + s = sha512(self.a_str) + s2 = s.copy() + s.update(self.b_str) + s2.update(self.c_str) + self.assertEqual( + "3fa253e7b093d5bc7b31f613f03833a4d39341cf73642349a46f26b39b5d95c97bb4e16fc588bda81d5c7a2db62cfca5c4c71a142cf02fd78409bffe5e4f408c", + s.hexdigest(), + ) + self.assertEqual( + "2e4d68ec2d2836f24718b24442db027141fd2f7e06fb11c1460b013017feb0e74dea9d9415abe51b729ad86792bd5cd2cec9567d58a47a03785028376e7a5cc1", + s2.hexdigest(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/python-stdlib/hmac/hmac.py b/python-stdlib/hmac/hmac.py index 28631042f..dbbdd4718 100644 --- a/python-stdlib/hmac/hmac.py +++ b/python-stdlib/hmac/hmac.py @@ -17,7 +17,7 @@ def __init__(self, key, msg=None, digestmod=None): make_hash = digestmod # A elif isinstance(digestmod, str): # A hash name suitable for hashlib.new(). - make_hash = lambda d=b"": hashlib.new(digestmod, d) # B + make_hash = lambda d=b"": getattr(hashlib, digestmod)(d) else: # A module supporting PEP 247. make_hash = digestmod.new # C diff --git a/python-stdlib/hmac/manifest.py b/python-stdlib/hmac/manifest.py index 92b8b18e4..ff0a62f08 100644 --- a/python-stdlib/hmac/manifest.py +++ b/python-stdlib/hmac/manifest.py @@ -1,3 +1,3 @@ -metadata(version="3.4.2-3") +metadata(version="3.4.4") module("hmac.py") diff --git a/python-stdlib/hmac/test_hmac.py b/python-stdlib/hmac/test_hmac.py index d155dd6a2..1cfcf4e37 100644 --- a/python-stdlib/hmac/test_hmac.py +++ b/python-stdlib/hmac/test_hmac.py @@ -8,7 +8,7 @@ msg = b"zlutoucky kun upel dabelske ody" -dig = hmac.new(b"1234567890", msg=msg, digestmod=hashlib.sha256).hexdigest() +dig = hmac.new(b"1234567890", msg=msg, digestmod="sha256").hexdigest() print("c735e751e36b08fb01e25794bdb15e7289b82aecdb652c8f4f72f307b39dad39") print(dig) diff --git a/python-stdlib/html/manifest.py b/python-stdlib/html/manifest.py index 2f0dcec77..c5705dd4b 100644 --- a/python-stdlib/html/manifest.py +++ b/python-stdlib/html/manifest.py @@ -1,4 +1,4 @@ -metadata(version="3.3.3-2") +metadata(version="3.3.4") require("string") diff --git a/python-stdlib/inspect/inspect.py b/python-stdlib/inspect/inspect.py index 06aba8762..c16c6b3e3 100644 --- a/python-stdlib/inspect/inspect.py +++ b/python-stdlib/inspect/inspect.py @@ -1,5 +1,7 @@ import sys +_g = lambda: (yield) + def getmembers(obj, pred=None): res = [] @@ -16,11 +18,16 @@ def isfunction(obj): def isgeneratorfunction(obj): - return isinstance(obj, type(lambda: (yield))) + return isinstance(obj, type(_g)) def isgenerator(obj): - return isinstance(obj, type(lambda: (yield)())) + return isinstance(obj, type((_g)())) + + +# In MicroPython there's currently no way to distinguish between generators and coroutines. +iscoroutinefunction = isgeneratorfunction +iscoroutine = isgenerator class _Class: diff --git a/python-stdlib/inspect/manifest.py b/python-stdlib/inspect/manifest.py index a9d5a2381..e99e659f2 100644 --- a/python-stdlib/inspect/manifest.py +++ b/python-stdlib/inspect/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1.2") +metadata(version="0.1.3") module("inspect.py") diff --git a/python-stdlib/inspect/test_inspect.py b/python-stdlib/inspect/test_inspect.py new file mode 100644 index 000000000..29ed80f11 --- /dev/null +++ b/python-stdlib/inspect/test_inspect.py @@ -0,0 +1,60 @@ +import inspect +import unittest + + +def fun(): + return 1 + + +def gen(): + yield 1 + + +class Class: + def meth(self): + pass + + +entities = ( + fun, + gen, + gen(), + Class, + Class.meth, + Class().meth, + inspect, +) + + +class TestInspect(unittest.TestCase): + def _test_is_helper(self, f, *entities_true): + for entity in entities: + result = f(entity) + if entity in entities_true: + self.assertTrue(result) + else: + self.assertFalse(result) + + def test_isfunction(self): + self._test_is_helper(inspect.isfunction, entities[0], entities[4]) + + def test_isgeneratorfunction(self): + self._test_is_helper(inspect.isgeneratorfunction, entities[1]) + + def test_isgenerator(self): + self._test_is_helper(inspect.isgenerator, entities[2]) + + def test_iscoroutinefunction(self): + self._test_is_helper(inspect.iscoroutinefunction, entities[1]) + + def test_iscoroutine(self): + self._test_is_helper(inspect.iscoroutine, entities[2]) + + def test_ismethod(self): + self._test_is_helper(inspect.ismethod, entities[5]) + + def test_isclass(self): + self._test_is_helper(inspect.isclass, entities[3]) + + def test_ismodule(self): + self._test_is_helper(inspect.ismodule, entities[6]) diff --git a/python-stdlib/io/manifest.py b/python-stdlib/io/manifest.py index ba1659ce0..62c0a5147 100644 --- a/python-stdlib/io/manifest.py +++ b/python-stdlib/io/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1") +metadata(version="0.1.0") module("io.py") diff --git a/python-stdlib/keyword/manifest.py b/python-stdlib/keyword/manifest.py new file mode 100644 index 000000000..aad27ec89 --- /dev/null +++ b/python-stdlib/keyword/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.0.1") + +module("keyword.py") diff --git a/python-stdlib/logging/logging.py b/python-stdlib/logging/logging.py index 1a5668476..f4874df7d 100644 --- a/python-stdlib/logging/logging.py +++ b/python-stdlib/logging/logging.py @@ -1,5 +1,5 @@ from micropython import const - +import io import sys import time @@ -58,6 +58,7 @@ def format(self, record): class StreamHandler(Handler): def __init__(self, stream=None): + super().__init__() self.stream = _stream if stream is None else stream self.terminator = "\n" @@ -148,10 +149,17 @@ def error(self, msg, *args): def critical(self, msg, *args): self.log(CRITICAL, msg, *args) - def exception(self, msg, *args): + def exception(self, msg, *args, exc_info=True): self.log(ERROR, msg, *args) - if hasattr(sys, "exc_info"): - sys.print_exception(sys.exc_info()[1], _stream) + tb = None + if isinstance(exc_info, BaseException): + tb = exc_info + elif hasattr(sys, "exc_info"): + tb = sys.exc_info()[1] + if tb: + buf = io.StringIO() + sys.print_exception(tb, buf) + self.log(ERROR, buf.getvalue()) def addHandler(self, handler): self.handlers.append(handler) diff --git a/python-stdlib/logging/manifest.py b/python-stdlib/logging/manifest.py index 68fc2db09..d9f0ee886 100644 --- a/python-stdlib/logging/manifest.py +++ b/python-stdlib/logging/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.5") +metadata(version="0.6.1") module("logging.py") diff --git a/python-stdlib/os-path/manifest.py b/python-stdlib/os-path/manifest.py index fd1885223..4433e6a4d 100644 --- a/python-stdlib/os-path/manifest.py +++ b/python-stdlib/os-path/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.1.4") +metadata(version="0.2.0") # Originally written by Paul Sokolovsky. diff --git a/python-stdlib/os-path/os/path.py b/python-stdlib/os-path/os/path.py index 7b4f937e5..b9ae1972f 100644 --- a/python-stdlib/os-path/os/path.py +++ b/python-stdlib/os-path/os/path.py @@ -66,6 +66,13 @@ def isdir(path): return False +def isfile(path): + try: + return bool(os.stat(path)[0] & 0x8000) + except OSError: + return False + + def expanduser(s): if s == "~" or s.startswith("~/"): h = os.getenv("HOME") diff --git a/python-stdlib/os-path/test_path.py b/python-stdlib/os-path/test_path.py index d2d3a3be4..85178364b 100644 --- a/python-stdlib/os-path/test_path.py +++ b/python-stdlib/os-path/test_path.py @@ -20,3 +20,7 @@ assert isdir(dir + "/os") assert not isdir(dir + "/os--") assert not isdir(dir + "/test_path.py") + +assert not isfile(dir + "/os") +assert isfile(dir + "/test_path.py") +assert not isfile(dir + "/test_path.py--") diff --git a/python-stdlib/os/manifest.py b/python-stdlib/os/manifest.py index 7cdeee65a..cd59f0c91 100644 --- a/python-stdlib/os/manifest.py +++ b/python-stdlib/os/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.6") +metadata(version="0.6.0") # Originally written by Paul Sokolovsky. diff --git a/python-stdlib/pathlib/pathlib.py b/python-stdlib/pathlib/pathlib.py index d01d81d32..e0f961373 100644 --- a/python-stdlib/pathlib/pathlib.py +++ b/python-stdlib/pathlib/pathlib.py @@ -47,6 +47,9 @@ def __init__(self, *segments): def __truediv__(self, other): return Path(self._path, str(other)) + def __rtruediv__(self, other): + return Path(other, self._path) + def __repr__(self): return f'{type(self).__name__}("{self._path}")' diff --git a/python-stdlib/pathlib/tests/test_pathlib.py b/python-stdlib/pathlib/tests/test_pathlib.py index c52cd9705..e632e1242 100644 --- a/python-stdlib/pathlib/tests/test_pathlib.py +++ b/python-stdlib/pathlib/tests/test_pathlib.py @@ -322,3 +322,14 @@ def test_with_suffix(self): self.assertTrue(Path("foo/test").with_suffix(".tar") == Path("foo/test.tar")) self.assertTrue(Path("foo/bar.bin").with_suffix(".txt") == Path("foo/bar.txt")) self.assertTrue(Path("bar.txt").with_suffix("") == Path("bar")) + + def test_rtruediv(self): + """Works as of micropython ea7031f""" + res = "foo" / Path("bar") + self.assertTrue(res == Path("foo/bar")) + + def test_rtruediv_inplace(self): + """Works as of micropython ea7031f""" + res = "foo" + res /= Path("bar") + self.assertTrue(res == Path("foo/bar")) diff --git a/python-stdlib/pickle/manifest.py b/python-stdlib/pickle/manifest.py index 0f6fbf766..412373a33 100644 --- a/python-stdlib/pickle/manifest.py +++ b/python-stdlib/pickle/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1") +metadata(version="0.1.0") module("pickle.py") diff --git a/python-stdlib/pkg_resources/pkg_resources.py b/python-stdlib/pkg_resources/pkg_resources.py index cd3e0fe96..d69cb0577 100644 --- a/python-stdlib/pkg_resources/pkg_resources.py +++ b/python-stdlib/pkg_resources/pkg_resources.py @@ -1,4 +1,4 @@ -import uio +import io c = {} @@ -18,11 +18,11 @@ def resource_stream(package, resource): else: d = "." # if d[0] != "/": - # import uos - # d = uos.getcwd() + "/" + d + # import os + # d = os.getcwd() + "/" + d c[package] = d + "/" p = c[package] if isinstance(p, dict): - return uio.BytesIO(p[resource]) + return io.BytesIO(p[resource]) return open(p + resource, "rb") diff --git a/python-stdlib/quopri/test_quopri.py b/python-stdlib/quopri/test_quopri.py index 5655dd8b0..b87e54842 100644 --- a/python-stdlib/quopri/test_quopri.py +++ b/python-stdlib/quopri/test_quopri.py @@ -1,7 +1,6 @@ -from test import support import unittest -import sys, os, io, subprocess +import sys, os, io import quopri @@ -193,7 +192,8 @@ def test_decode_header(self): for p, e in self.HSTRINGS: self.assertEqual(quopri.decodestring(e, header=True), p) - def _test_scriptencode(self): + @unittest.skip("requires subprocess") + def test_scriptencode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen( [sys.executable, "-mquopri"], stdin=subprocess.PIPE, stdout=subprocess.PIPE @@ -210,7 +210,8 @@ def _test_scriptencode(self): self.assertEqual(cout[i], e[i]) self.assertEqual(cout, e) - def _test_scriptdecode(self): + @unittest.skip("requires subprocess") + def test_scriptdecode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen( [sys.executable, "-mquopri", "-d"], stdin=subprocess.PIPE, stdout=subprocess.PIPE @@ -220,11 +221,3 @@ def _test_scriptdecode(self): cout = cout.decode("latin-1") p = p.decode("latin-1") self.assertEqual(cout.splitlines(), p.splitlines()) - - -def test_main(): - support.run_unittest(QuopriTestCase) - - -if __name__ == "__main__": - test_main() diff --git a/python-stdlib/random/manifest.py b/python-stdlib/random/manifest.py index fcd10007d..e09c8b0f7 100644 --- a/python-stdlib/random/manifest.py +++ b/python-stdlib/random/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.2") +metadata(version="0.2.0") module("random.py") diff --git a/python-stdlib/ssl/manifest.py b/python-stdlib/ssl/manifest.py index fb92cdf48..a99523071 100644 --- a/python-stdlib/ssl/manifest.py +++ b/python-stdlib/ssl/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1") +metadata(version="0.2.1") -module("ssl.py") +module("ssl.py", opt=3) diff --git a/python-stdlib/ssl/ssl.py b/python-stdlib/ssl/ssl.py index 9262f5fb5..c61904be7 100644 --- a/python-stdlib/ssl/ssl.py +++ b/python-stdlib/ssl/ssl.py @@ -1,36 +1,65 @@ -from ussl import * -import ussl as _ussl +import tls +from tls import * -# Constants -for sym in "CERT_NONE", "CERT_OPTIONAL", "CERT_REQUIRED": - if sym not in globals(): - globals()[sym] = object() + +class SSLContext: + def __init__(self, *args): + self._context = tls.SSLContext(*args) + self._context.verify_mode = CERT_NONE + + @property + def verify_mode(self): + return self._context.verify_mode + + @verify_mode.setter + def verify_mode(self, val): + self._context.verify_mode = val + + def load_cert_chain(self, certfile, keyfile): + if isinstance(certfile, str): + with open(certfile, "rb") as f: + certfile = f.read() + if isinstance(keyfile, str): + with open(keyfile, "rb") as f: + keyfile = f.read() + self._context.load_cert_chain(certfile, keyfile) + + def load_verify_locations(self, cafile=None, cadata=None): + if cafile: + with open(cafile, "rb") as f: + cadata = f.read() + self._context.load_verify_locations(cadata) + + def wrap_socket( + self, sock, server_side=False, do_handshake_on_connect=True, server_hostname=None + ): + return self._context.wrap_socket( + sock, + server_side=server_side, + do_handshake_on_connect=do_handshake_on_connect, + server_hostname=server_hostname, + ) def wrap_socket( sock, - keyfile=None, - certfile=None, server_side=False, + key=None, + cert=None, cert_reqs=CERT_NONE, - *, - ca_certs=None, - server_hostname=None + cadata=None, + server_hostname=None, + do_handshake=True, ): - # TODO: More arguments accepted by CPython could also be handled here. - # That would allow us to accept ca_certs as a positional argument, which - # we should. - kw = {} - if keyfile is not None: - kw["keyfile"] = keyfile - if certfile is not None: - kw["certfile"] = certfile - if server_side is not False: - kw["server_side"] = server_side - if cert_reqs is not CERT_NONE: - kw["cert_reqs"] = cert_reqs - if ca_certs is not None: - kw["ca_certs"] = ca_certs - if server_hostname is not None: - kw["server_hostname"] = server_hostname - return _ussl.wrap_socket(sock, **kw) + con = SSLContext(PROTOCOL_TLS_SERVER if server_side else PROTOCOL_TLS_CLIENT) + if cert or key: + con.load_cert_chain(cert, key) + if cadata: + con.load_verify_locations(cadata=cadata) + con.verify_mode = cert_reqs + return con.wrap_socket( + sock, + server_side=server_side, + do_handshake_on_connect=do_handshake, + server_hostname=server_hostname, + ) diff --git a/python-stdlib/tarfile-write/example-append.py b/python-stdlib/tarfile-write/example-append.py new file mode 100644 index 000000000..f496eb3aa --- /dev/null +++ b/python-stdlib/tarfile-write/example-append.py @@ -0,0 +1,15 @@ +""" tar append writes additional files to the end of an existing tar file.""" +import os +import sys +import tarfile + +if len(sys.argv) < 2: + raise ValueError("Usage: %s appendfile.tar newinputfile1 ..." % sys.argv[0]) + +tarfile = sys.argv[1] +if not tarfile.endswith(".tar"): + raise ValueError("Filename %s does not end with .tar" % tarfile) + +with tarfile.TarFile(sys.argv[1], "a") as t: + for filename in sys.argv[2:]: + t.add(filename) diff --git a/python-stdlib/tarfile-write/example-create.py b/python-stdlib/tarfile-write/example-create.py new file mode 100644 index 000000000..ee6ec6255 --- /dev/null +++ b/python-stdlib/tarfile-write/example-create.py @@ -0,0 +1,14 @@ +""" tar create writes a new tar file containing the specified files.""" +import sys +import tarfile + +if len(sys.argv) < 2: + raise ValueError("Usage: %s outputfile.tar inputfile1 ..." % sys.argv[0]) + +tarfile = sys.argv[1] +if not tarfile.endswith(".tar"): + raise ValueError("Filename %s does not end with .tar" % tarfile) + +with tarfile.TarFile(sys.argv[1], "w") as t: + for filename in sys.argv[2:]: + t.add(filename) diff --git a/python-stdlib/tarfile-write/manifest.py b/python-stdlib/tarfile-write/manifest.py new file mode 100644 index 000000000..bc4f37741 --- /dev/null +++ b/python-stdlib/tarfile-write/manifest.py @@ -0,0 +1,4 @@ +metadata(description="Adds write (create/append) support to tarfile.", version="0.1.2") + +require("tarfile") +package("tarfile") diff --git a/python-stdlib/tarfile-write/tarfile/write.py b/python-stdlib/tarfile-write/tarfile/write.py new file mode 100644 index 000000000..527b3317b --- /dev/null +++ b/python-stdlib/tarfile-write/tarfile/write.py @@ -0,0 +1,121 @@ +"""Additions to the TarFile class to support creating and appending tar files. + +The methods defined below in are injected into the TarFile class in the +tarfile package. +""" + +import uctypes +import os + +# Extended subset of tar header fields including the ones we'll write. +# http://www.gnu.org/software/tar/manual/html_node/Standard.html +_TAR_HEADER = { + "name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100), + "mode": (uctypes.ARRAY | 100, uctypes.UINT8 | 8), + "uid": (uctypes.ARRAY | 108, uctypes.UINT8 | 8), + "gid": (uctypes.ARRAY | 116, uctypes.UINT8 | 8), + "size": (uctypes.ARRAY | 124, uctypes.UINT8 | 12), + "mtime": (uctypes.ARRAY | 136, uctypes.UINT8 | 12), + "chksum": (uctypes.ARRAY | 148, uctypes.UINT8 | 8), + "typeflag": (uctypes.ARRAY | 156, uctypes.UINT8 | 1), +} + + +_NUL = const(b"\0") # the null character +_BLOCKSIZE = const(512) # length of processing blocks +_RECORDSIZE = const(_BLOCKSIZE * 20) # length of records + + +def _open_write(self, name, mode, fileobj): + if mode == "w": + if not fileobj: + self.f = open(name, "wb") + else: + self.f = fileobj + elif mode == "a": + if not fileobj: + self.f = open(name, "r+b") + else: + self.f = fileobj + # Read through the existing file. + while self.next(): + pass + # Position at start of end block. + self.f.seek(self.offset) + else: + raise ValueError("mode " + mode + " not supported.") + + +def _close_write(self): + # Must be called to complete writing a tar file. + if self.mode == "w": + self.f.write(_NUL * (_BLOCKSIZE * 2)) + self.offset += _BLOCKSIZE * 2 + remainder = self.offset % _RECORDSIZE + if remainder: + self.f.write(_NUL * (_RECORDSIZE - remainder)) + + +def addfile(self, tarinfo, fileobj=None): + # Write the header: 100 bytes of name, 8 bytes of mode in octal... + buf = bytearray(_BLOCKSIZE) + name = tarinfo.name + size = tarinfo.size + if tarinfo.isdir(): + size = 0 + if not name.endswith("/"): + name += "/" + hdr = uctypes.struct(uctypes.addressof(buf), _TAR_HEADER, uctypes.LITTLE_ENDIAN) + hdr.name[:] = name.encode("utf-8")[:100] + hdr.mode[:] = b"%07o\0" % ((0o755 if tarinfo.isdir() else 0o644) & 0o7777) + hdr.uid[:] = b"%07o\0" % tarinfo.uid + hdr.gid[:] = b"%07o\0" % tarinfo.gid + hdr.size[:] = b"%011o\0" % size + hdr.mtime[:] = b"%011o\0" % tarinfo.mtime + hdr.typeflag[:] = b"5" if tarinfo.isdir() else b"0" + # Checksum is calculated with checksum field all blanks. + hdr.chksum[:] = b" " + # Calculate and insert the actual checksum. + chksum = sum(buf) + hdr.chksum[:] = b"%06o\0 " % chksum + # Emit the header. + self.f.write(buf) + self.offset += len(buf) + + # Copy the file contents, if any. + if fileobj: + n_bytes = self.f.write(fileobj.read()) + self.offset += n_bytes + remains = -n_bytes & (_BLOCKSIZE - 1) # == 0b111111111 + if remains: + buf = bytearray(remains) + self.f.write(buf) + self.offset += len(buf) + + +def add(self, name, recursive=True): + from . import TarInfo + + try: + stat = os.stat(name) + res_name = (name + '/') if (stat[0] & 0xf000) == 0x4000 else name + tarinfo = TarInfo(res_name) + tarinfo.mode = stat[0] + tarinfo.uid = stat[4] + tarinfo.gid = stat[5] + tarinfo.size = stat[6] + tarinfo.mtime = stat[8] + except OSError: + print("Cannot stat", name, " - skipping.") + return + if not (tarinfo.isdir() or tarinfo.isreg()): + # We only accept directories or regular files. + print(name, "is not a directory or regular file - skipping.") + return + if tarinfo.isdir(): + self.addfile(tarinfo) + if recursive: + for f in os.ilistdir(name): + self.add(name + "/" + f[0], recursive) + else: # type == REGTYPE + self.addfile(tarinfo, open(name, "rb")) diff --git a/python-stdlib/tarfile/example-extract.py b/python-stdlib/tarfile/example-extract.py new file mode 100644 index 000000000..94ce829ce --- /dev/null +++ b/python-stdlib/tarfile/example-extract.py @@ -0,0 +1,16 @@ +import sys +import os +import tarfile + +if len(sys.argv) < 2: + raise ValueError("Usage: %s inputfile.tar" % sys.argv[0]) + +t = tarfile.TarFile(sys.argv[1]) +for i in t: + print(i.name) + if i.type == tarfile.DIRTYPE: + os.mkdir(i.name) + else: + f = t.extractfile(i) + with open(i.name, "wb") as of: + of.write(f.read()) diff --git a/python-stdlib/tarfile/manifest.py b/python-stdlib/tarfile/manifest.py new file mode 100644 index 000000000..9940bb051 --- /dev/null +++ b/python-stdlib/tarfile/manifest.py @@ -0,0 +1,5 @@ +metadata(description="Read-only implementation of Python's tarfile.", version="0.4.1") + +# Originally written by Paul Sokolovsky. + +package("tarfile") diff --git a/python-stdlib/tarfile/tarfile/__init__.py b/python-stdlib/tarfile/tarfile/__init__.py new file mode 100644 index 000000000..4bb95af30 --- /dev/null +++ b/python-stdlib/tarfile/tarfile/__init__.py @@ -0,0 +1,148 @@ +"""Subset of cpython tarfile class methods needed to decode tar files.""" + +import uctypes + +# Minimal set of tar header fields for reading. +# http://www.gnu.org/software/tar/manual/html_node/Standard.html +# The "size" entry is 11 (not 12) to implicitly cut off the null terminator. +_TAR_HEADER = { + "name": (uctypes.ARRAY | 0, uctypes.UINT8 | 100), + "size": (uctypes.ARRAY | 124, uctypes.UINT8 | 11), +} + +DIRTYPE = const("dir") +REGTYPE = const("file") + +# Constants for TarInfo.isdir, isreg. +_S_IFMT = const(0o170000) +_S_IFREG = const(0o100000) +_S_IFDIR = const(0o040000) + +_BLOCKSIZE = const(512) # length of processing blocks + + +def _roundup(val, align): + return (val + align - 1) & ~(align - 1) + + +class FileSection: + def __init__(self, f, content_len, aligned_len): + self.f = f + self.content_len = content_len + self.align = aligned_len - content_len + + def read(self, sz=65536): + if self.content_len == 0: + return b"" + if sz > self.content_len: + sz = self.content_len + data = self.f.read(sz) + sz = len(data) + self.content_len -= sz + return data + + def readinto(self, buf): + if self.content_len == 0: + return 0 + if len(buf) > self.content_len: + buf = memoryview(buf)[: self.content_len] + sz = self.f.readinto(buf) + self.content_len -= sz + return sz + + def skip(self): + sz = self.content_len + self.align + if sz: + buf = bytearray(16) + while sz: + s = min(sz, 16) + self.f.readinto(buf, s) + sz -= s + + +class TarInfo: + def __init__(self, name=""): + self.name = name + self.mode = _S_IFDIR if self.name[-1] == "/" else _S_IFREG + + @property + def type(self): + return DIRTYPE if self.isdir() else REGTYPE + + def __str__(self): + return "TarInfo(%r, %s, %d)" % (self.name, self.type, self.size) + + def isdir(self): + return (self.mode & _S_IFMT) == _S_IFDIR + + def isreg(self): + return (self.mode & _S_IFMT) == _S_IFREG + + +class TarFile: + def __init__(self, name=None, mode="r", fileobj=None): + self.subf = None + self.mode = mode + self.offset = 0 + if mode == "r": + if fileobj: + self.f = fileobj + else: + self.f = open(name, "rb") + else: + try: + self._open_write(name=name, mode=mode, fileobj=fileobj) + except AttributeError: + raise NotImplementedError("Install tarfile-write") + + def __enter__(self): + return self + + def __exit__(self, unused_type, unused_value, unused_traceback): + self.close() + + def next(self): + if self.subf: + self.subf.skip() + buf = self.f.read(_BLOCKSIZE) + if not buf: + return None + + h = uctypes.struct(uctypes.addressof(buf), _TAR_HEADER, uctypes.LITTLE_ENDIAN) + + # Empty block means end of archive + if h.name[0] == 0: + return None + + # Update the offset once we're sure it's not the run-out. + self.offset += len(buf) + d = TarInfo(str(h.name, "utf-8").rstrip("\0")) + d.size = int(bytes(h.size), 8) + self.subf = d.subf = FileSection(self.f, d.size, _roundup(d.size, _BLOCKSIZE)) + self.offset += _roundup(d.size, _BLOCKSIZE) + return d + + def __iter__(self): + return self + + def __next__(self): + v = self.next() + if v is None: + raise StopIteration + return v + + def extractfile(self, tarinfo): + return tarinfo.subf + + def close(self): + try: + self._close_write() + except AttributeError: + pass + self.f.close() + + # Add additional methods to support write/append from the tarfile-write package. + try: + from .write import _open_write, _close_write, addfile, add + except ImportError: + pass diff --git a/python-stdlib/textwrap/manifest.py b/python-stdlib/textwrap/manifest.py index d43a54bba..e287ac395 100644 --- a/python-stdlib/textwrap/manifest.py +++ b/python-stdlib/textwrap/manifest.py @@ -1,3 +1,3 @@ -metadata(version="3.4.2-1") +metadata(version="3.4.3") module("textwrap.py") diff --git a/python-stdlib/threading/manifest.py b/python-stdlib/threading/manifest.py index 4db21dd1d..8106584be 100644 --- a/python-stdlib/threading/manifest.py +++ b/python-stdlib/threading/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1") +metadata(version="0.1.0") module("threading.py") diff --git a/python-stdlib/time/README.md b/python-stdlib/time/README.md new file mode 100644 index 000000000..f07517305 --- /dev/null +++ b/python-stdlib/time/README.md @@ -0,0 +1,45 @@ +# time + +This library _extends_ the built-in [MicroPython `time` +module](https://docs.micropython.org/en/latest/library/time.html#module-time) to +include +[`time.strftime()`](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior). + +`strftime()` is omitted from the built-in `time` module to conserve space. + +## Installation + +Use `mip` via `mpremote`: + +```bash +> mpremote mip install time +``` + +See [Package management](https://docs.micropython.org/en/latest/reference/packages.html) for more details on using `mip` and `mpremote`. + +## Common uses + +`strftime()` is used when using a loggging [Formatter +Object](https://docs.python.org/3/library/logging.html#formatter-objects) that +employs +[`asctime`](https://docs.python.org/3/library/logging.html#formatter-objects). + +For example: + +```python +logging.Formatter('%(asctime)s | %(name)s | %(levelname)s - %(message)s') +``` + +The expected output might look like: + +```text +Tue Feb 17 09:42:58 2009 | MAIN | INFO - test +``` + +But if this `time` extension library isn't installed, `asctime` will always be +`None`: + + +```text +None | MAIN | INFO - test +``` diff --git a/python-stdlib/time/manifest.py b/python-stdlib/time/manifest.py index 052ed4a17..71af915c4 100644 --- a/python-stdlib/time/manifest.py +++ b/python-stdlib/time/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.1") +metadata(version="0.1.0") module("time.py") diff --git a/python-stdlib/traceback/manifest.py b/python-stdlib/traceback/manifest.py index 65b3cdaa5..b3ef8343c 100644 --- a/python-stdlib/traceback/manifest.py +++ b/python-stdlib/traceback/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.3") +metadata(version="0.3.0") module("traceback.py") diff --git a/python-stdlib/types/manifest.py b/python-stdlib/types/manifest.py new file mode 100644 index 000000000..35a47d86c --- /dev/null +++ b/python-stdlib/types/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.0.1") + +module("types.py") diff --git a/python-stdlib/unittest-discover/manifest.py b/python-stdlib/unittest-discover/manifest.py index 87bd94ae0..5610f41e2 100644 --- a/python-stdlib/unittest-discover/manifest.py +++ b/python-stdlib/unittest-discover/manifest.py @@ -1,4 +1,4 @@ -metadata(version="0.1.1") +metadata(version="0.1.3") require("argparse") require("fnmatch") diff --git a/python-stdlib/unittest-discover/tests/sub/sub.py b/python-stdlib/unittest-discover/tests/sub/sub.py new file mode 100644 index 000000000..b6614dd63 --- /dev/null +++ b/python-stdlib/unittest-discover/tests/sub/sub.py @@ -0,0 +1 @@ +imported = True diff --git a/python-stdlib/unittest-discover/tests/sub/test_module_import.py b/python-stdlib/unittest-discover/tests/sub/test_module_import.py new file mode 100644 index 000000000..5c6404d6f --- /dev/null +++ b/python-stdlib/unittest-discover/tests/sub/test_module_import.py @@ -0,0 +1,13 @@ +import sys +import unittest + + +class TestModuleImport(unittest.TestCase): + def test_ModuleImportPath(self): + try: + from sub.sub import imported + assert imported + except ImportError: + print("This test is intended to be run with unittest discover" + "from the unittest-discover/tests dir. sys.path:", sys.path) + raise diff --git a/python-stdlib/unittest-discover/unittest/__main__.py b/python-stdlib/unittest-discover/unittest/__main__.py index e64c18c1b..09dfd03b9 100644 --- a/python-stdlib/unittest-discover/unittest/__main__.py +++ b/python-stdlib/unittest-discover/unittest/__main__.py @@ -6,7 +6,12 @@ from fnmatch import fnmatch from micropython import const -from unittest import TestRunner, TestResult, TestSuite +try: + from unittest import TestRunner, TestResult, TestSuite +except ImportError: + print("Error: This must be used from an installed copy of unittest-discover which will" + " also install base unittest module.") + raise # Run a single test in a clean environment. @@ -14,11 +19,11 @@ def _run_test_module(runner: TestRunner, module_name: str, *extra_paths: list[st module_snapshot = {k: v for k, v in sys.modules.items()} path_snapshot = sys.path[:] try: - for path in reversed(extra_paths): + for path in extra_paths: if path: sys.path.insert(0, path) - module = __import__(module_name) + module = __import__(module_name, None, None, module_name) suite = TestSuite(module_name) suite._load_module(module) return runner.run(suite) @@ -36,16 +41,18 @@ def _run_all_in_dir(runner: TestRunner, path: str, pattern: str, top: str): for fname, ftype, *_ in os.ilistdir(path): if fname in ("..", "."): continue + fpath = "/".join((path, fname)) if ftype == _DIR_TYPE: result += _run_all_in_dir( runner=runner, - path="/".join((path, fname)), + path=fpath, pattern=pattern, top=top, ) if fnmatch(fname, pattern): - module_name = fname.rsplit(".", 1)[0] - result += _run_test_module(runner, module_name, path, top) + module_path = fpath.rsplit(".", 1)[0] # remove ext + module_path = module_path.replace("/", ".").strip(".") + result += _run_test_module(runner, module_path, top) return result @@ -111,7 +118,6 @@ def _dirname_filename_no_ext(path): def discover_main(): - failures = 0 runner = TestRunner() if len(sys.argv) == 1 or ( @@ -121,22 +127,27 @@ def discover_main(): ): # No args, or `python -m unittest discover ...`. result = _discover(runner) - failures += result.failuresNum or result.errorsNum else: + result = TestResult() for test_spec in sys.argv[1:]: try: os.stat(test_spec) # File exists, strip extension and import with its parent directory in sys.path. dirname, module_name = _dirname_filename_no_ext(test_spec) - result = _run_test_module(runner, module_name, dirname) + res = _run_test_module(runner, module_name, dirname) except OSError: # Not a file, treat as named module to import. - result = _run_test_module(runner, test_spec) + res = _run_test_module(runner, test_spec) - failures += result.failuresNum or result.errorsNum + result += res + + if not result.testsRun: + # If tests are run their results are already printed. + # Ensure an appropriate output is printed if no tests are found. + runner.run(TestSuite()) # Terminate with non zero return code in case of failures. - sys.exit(failures) + sys.exit(result.failuresNum + result.errorsNum) discover_main() diff --git a/python-stdlib/unittest/manifest.py b/python-stdlib/unittest/manifest.py index 8e419b63c..a01bbb8e6 100644 --- a/python-stdlib/unittest/manifest.py +++ b/python-stdlib/unittest/manifest.py @@ -1,3 +1,3 @@ -metadata(version="0.10.2") +metadata(version="0.10.4") package("unittest") diff --git a/python-stdlib/unittest/tests/test_exception.py b/python-stdlib/unittest/tests/exception.py similarity index 66% rename from python-stdlib/unittest/tests/test_exception.py rename to python-stdlib/unittest/tests/exception.py index 470ffdcc2..0e828e226 100644 --- a/python-stdlib/unittest/tests/test_exception.py +++ b/python-stdlib/unittest/tests/exception.py @@ -1,3 +1,5 @@ +# This makes unittest return an error code, so is not named "test_xxx.py". + import unittest diff --git a/python-stdlib/unittest/tests/test_subtest.py b/python-stdlib/unittest/tests/test_subtest.py new file mode 100644 index 000000000..324150e27 --- /dev/null +++ b/python-stdlib/unittest/tests/test_subtest.py @@ -0,0 +1,14 @@ +import unittest + + +class Test(unittest.TestCase): + def test_subtest_skip(self): + for i in range(4): + with self.subTest(i=i): + print("sub test", i) + if i == 2: + self.skipTest("skip 2") + + +if __name__ == "__main__": + unittest.main() diff --git a/python-stdlib/unittest/unittest/__init__.py b/python-stdlib/unittest/unittest/__init__.py index ac24e7535..61b315788 100644 --- a/python-stdlib/unittest/unittest/__init__.py +++ b/python-stdlib/unittest/unittest/__init__.py @@ -198,7 +198,7 @@ def assertRaises(self, exc, func=None, *args, **kwargs): except Exception as e: if isinstance(e, exc): return - raise + raise e assert False, "%r not raised" % exc @@ -300,9 +300,10 @@ def wasSuccessful(self): return self.errorsNum == 0 and self.failuresNum == 0 def printErrors(self): - print() - self.printErrorList(self.errors) - self.printErrorList(self.failures) + if self.errors or self.failures: + print() + self.printErrorList(self.errors) + self.printErrorList(self.failures) def printErrorList(self, lst): sep = "----------------------------------------------------------------------" @@ -347,7 +348,13 @@ def _handle_test_exception( exc = exc_info[1] traceback = exc_info[2] ex_str = _capture_exc(exc, traceback) - if isinstance(exc, AssertionError): + if isinstance(exc, SkipTest): + reason = exc.args[0] + test_result.skippedNum += 1 + test_result.skipped.append((current_test, reason)) + print(" skipped:", reason) + return + elif isinstance(exc, AssertionError): test_result.failuresNum += 1 test_result.failures.append((current_test, ex_str)) if verbose: @@ -395,17 +402,12 @@ def run_one(test_function): print(" FAIL") else: print(" ok") - except SkipTest as e: - reason = e.args[0] - print(" skipped:", reason) - test_result.skippedNum += 1 - test_result.skipped.append((name, c, reason)) except Exception as ex: _handle_test_exception( current_test=(name, c), test_result=test_result, exc_info=(type(ex), ex, None) ) # Uncomment to investigate failure in detail - # raise + # raise ex finally: __test_result__ = None __current_test__ = None diff --git a/python-stdlib/zlib/manifest.py b/python-stdlib/zlib/manifest.py new file mode 100644 index 000000000..f95602f25 --- /dev/null +++ b/python-stdlib/zlib/manifest.py @@ -0,0 +1,3 @@ +metadata(version="1.0.0", description="Compression and decompression using the deflate algorithm") + +module("zlib.py") diff --git a/python-stdlib/zlib/zlib.py b/python-stdlib/zlib/zlib.py new file mode 100644 index 000000000..e6c342ef7 --- /dev/null +++ b/python-stdlib/zlib/zlib.py @@ -0,0 +1,39 @@ +# MicroPython zlib module +# MIT license; Copyright (c) 2023 Jim Mussared + +import io, deflate + +_MAX_WBITS = const(15) + + +def _decode_wbits(wbits, decompress): + if -15 <= wbits <= -5: + return ( + deflate.RAW, + -wbits, + ) + elif 5 <= wbits <= 15: + return (deflate.ZLIB, wbits) + elif decompress and wbits == 0: + return (deflate.ZLIB,) + elif 21 <= wbits <= 31: + return (deflate.GZIP, wbits - 16) + elif decompress and 35 <= wbits <= 47: + return (deflate.AUTO, wbits - 32) + else: + raise ValueError("wbits") + + +if hasattr(deflate.DeflateIO, "write"): + + def compress(data, wbits=_MAX_WBITS): + f = io.BytesIO() + with deflate.DeflateIO(f, *_decode_wbits(wbits, False)) as g: + g.write(data) + return f.getvalue() + + +def decompress(data, wbits=_MAX_WBITS): + f = io.BytesIO(data) + with deflate.DeflateIO(f, *_decode_wbits(wbits, True)) as g: + return g.read() diff --git a/tools/build.py b/tools/build.py index 305870be7..442cf2121 100755 --- a/tools/build.py +++ b/tools/build.py @@ -64,7 +64,7 @@ # index.json is: # { -# "v": 1, <-- file format version +# "v": 2, <-- file format version # "updated": , # "packages": { # { @@ -78,7 +78,9 @@ # "7": ["0.2", "0.3", "0.4"], # ... <-- Other bytecode versions # "py": ["0.1", "0.2", "0.3", "0.4"] -# } +# }, +# // The following entries were added in file format version 2. +# path: "micropython/bluetooth/aioble", # }, # ... # } @@ -112,7 +114,6 @@ # mip (or other tools) should request /package/{mpy_version}/{package_name}/{version}.json. -import argparse import glob import hashlib import json @@ -123,7 +124,7 @@ import time -_JSON_VERSION_INDEX = 1 +_JSON_VERSION_INDEX = 2 _JSON_VERSION_PACKAGE = 1 @@ -132,7 +133,7 @@ # Create all directories in the path (such that the file can be created). -def _ensure_path_exists(file_path): +def ensure_path_exists(file_path): path = os.path.dirname(file_path) if not os.path.isdir(path): os.makedirs(path) @@ -155,7 +156,7 @@ def _identical_files(path_a, path_b): # Helper to write the object as json to the specified path, creating any # directories as required. def _write_json(obj, path, minify=False): - _ensure_path_exists(path) + ensure_path_exists(path) with open(path, "w") as f: json.dump( obj, f, indent=(None if minify else 2), separators=((",", ":") if minify else None) @@ -173,7 +174,7 @@ def _write_package_json( # Format s with bold red. -def _error_color(s): +def error_color(s): return _COLOR_ERROR_ON + s + _COLOR_ERROR_OFF @@ -191,7 +192,7 @@ def _write_hashed_file(package_name, src, target_path, out_file_dir, hash_prefix # that it's actually the same file. if not _identical_files(src.name, output_file_path): print( - _error_color("Hash collision processing:"), + error_color("Hash collision processing:"), package_name, file=sys.stderr, ) @@ -204,7 +205,7 @@ def _write_hashed_file(package_name, src, target_path, out_file_dir, hash_prefix sys.exit(1) else: # Create new file. - _ensure_path_exists(output_file_path) + ensure_path_exists(output_file_path) shutil.copyfile(src.name, output_file_path) return short_file_hash @@ -235,7 +236,7 @@ def _compile_as_mpy( ) except mpy_cross.CrossCompileError as e: print( - _error_color("Error:"), + error_color("Error:"), "Unable to compile", target_path, "in package", @@ -269,7 +270,7 @@ def _copy_as_py( # Update to the latest metadata, and add any new versions to the package in # the index json. -def _update_index_package_metadata(index_package_json, metadata, mpy_version): +def _update_index_package_metadata(index_package_json, metadata, mpy_version, package_path): index_package_json["version"] = metadata.version or "" index_package_json["author"] = "" # TODO: Make manifestfile.py capture this. index_package_json["description"] = metadata.description or "" @@ -284,6 +285,9 @@ def _update_index_package_metadata(index_package_json, metadata, mpy_version): print(" New version {}={}".format(v, metadata.version)) index_package_json["versions"][v].append(metadata.version) + # The following entries were added in file format version 2. + index_package_json["path"] = package_path + def build(output_path, hash_prefix_len, mpy_cross_path): import manifestfile @@ -319,7 +323,8 @@ def build(output_path, hash_prefix_len, mpy_cross_path): for lib_dir in lib_dirs: for manifest_path in glob.glob(os.path.join(lib_dir, "**", "manifest.py"), recursive=True): - print("{}".format(os.path.dirname(manifest_path))) + package_path = os.path.dirname(manifest_path) + print("{}".format(package_path)) # .../foo/manifest.py -> foo package_name = os.path.basename(os.path.dirname(manifest_path)) @@ -329,7 +334,7 @@ def build(output_path, hash_prefix_len, mpy_cross_path): # Append this package to the index. if not manifest.metadata().version: - print(_error_color("Warning:"), package_name, "doesn't have a version.") + print(error_color("Warning:"), package_name, "doesn't have a version.") # Try to find this package in the previous index.json. for p in index_json["packages"]: @@ -343,7 +348,9 @@ def build(output_path, hash_prefix_len, mpy_cross_path): } index_json["packages"].append(index_package_json) - _update_index_package_metadata(index_package_json, manifest.metadata(), mpy_version) + _update_index_package_metadata( + index_package_json, manifest.metadata(), mpy_version, package_path + ) # This is the package json that mip/mpremote downloads. mpy_package_json = { @@ -360,11 +367,12 @@ def build(output_path, hash_prefix_len, mpy_cross_path): for result in manifest.files(): # This isn't allowed in micropython-lib anyway. if result.file_type != manifestfile.FILE_TYPE_LOCAL: - print("Non-local file not supported.", file=sys.stderr) + print(error_color("Error:"), "Non-local file not supported.", file=sys.stderr) sys.exit(1) if not result.target_path.endswith(".py"): print( + error_color("Error:"), "Target path isn't a .py file:", result.target_path, file=sys.stderr, diff --git a/tools/ci.sh b/tools/ci.sh index 75c8791d5..6689e8aa4 100755 --- a/tools/ci.sh +++ b/tools/ci.sh @@ -1,18 +1,109 @@ #!/bin/bash +CP=/bin/cp + +######################################################################################## +# commit formatting + +function ci_commit_formatting_run { + git remote add upstream https://github.com/micropython/micropython-lib.git + git fetch --depth=100 upstream master + # If the common ancestor commit hasn't been found, fetch more. + git merge-base upstream/master HEAD || git fetch --unshallow upstream master + # For a PR, upstream/master..HEAD ends with a merge commit into master, exclude that one. + tools/verifygitlog.py -v upstream/master..HEAD --no-merges +} + ######################################################################################## -# code formatting +# package tests -function ci_code_formatting_setup { - sudo apt-add-repository --yes --update ppa:pybricks/ppa - sudo apt-get install uncrustify - pip3 install black - uncrustify --version - black --version +MICROPYTHON=/tmp/micropython/ports/unix/build-standard/micropython + +function ci_package_tests_setup_micropython { + git clone https://github.com/micropython/micropython.git /tmp/micropython + + # build mpy-cross and micropython (use -O0 to speed up the build) + make -C /tmp/micropython/mpy-cross -j CFLAGS_EXTRA=-O0 + make -C /tmp/micropython/ports/unix submodules + make -C /tmp/micropython/ports/unix -j CFLAGS_EXTRA=-O0 } -function ci_code_formatting_run { - tools/codeformat.py -v +function ci_package_tests_setup_lib { + mkdir -p ~/.micropython/lib + $CP micropython/ucontextlib/ucontextlib.py ~/.micropython/lib/ + $CP python-stdlib/fnmatch/fnmatch.py ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-core/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha224/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha256/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha384/hashlib ~/.micropython/lib/ + $CP -r python-stdlib/hashlib-sha512/hashlib ~/.micropython/lib/ + $CP python-stdlib/shutil/shutil.py ~/.micropython/lib/ + $CP python-stdlib/tempfile/tempfile.py ~/.micropython/lib/ + $CP -r python-stdlib/unittest/unittest ~/.micropython/lib/ + $CP -r python-stdlib/unittest-discover/unittest ~/.micropython/lib/ + $CP unix-ffi/ffilib/ffilib.py ~/.micropython/lib/ + tree ~/.micropython +} + +function ci_package_tests_run { + for test in \ + micropython/drivers/storage/sdcard/sdtest.py \ + micropython/xmltok/test_xmltok.py \ + python-ecosys/requests/test_requests.py \ + python-stdlib/argparse/test_argparse.py \ + python-stdlib/base64/test_base64.py \ + python-stdlib/binascii/test_binascii.py \ + python-stdlib/collections-defaultdict/test_defaultdict.py \ + python-stdlib/functools/test_partial.py \ + python-stdlib/functools/test_reduce.py \ + python-stdlib/heapq/test_heapq.py \ + python-stdlib/hmac/test_hmac.py \ + python-stdlib/itertools/test_itertools.py \ + python-stdlib/operator/test_operator.py \ + python-stdlib/os-path/test_path.py \ + python-stdlib/pickle/test_pickle.py \ + python-stdlib/string/test_translate.py \ + python-stdlib/unittest/tests/exception.py \ + unix-ffi/gettext/test_gettext.py \ + unix-ffi/pwd/test_getpwnam.py \ + unix-ffi/re/test_re.py \ + unix-ffi/sqlite3/test_sqlite3.py \ + unix-ffi/sqlite3/test_sqlite3_2.py \ + unix-ffi/sqlite3/test_sqlite3_3.py \ + unix-ffi/time/test_strftime.py \ + ; do + echo "Running test $test" + (cd `dirname $test` && $MICROPYTHON `basename $test`) + if [ $? -ne 0 ]; then + false # make this function return an error code + return + fi + done + + for path in \ + micropython/ucontextlib \ + python-stdlib/contextlib \ + python-stdlib/datetime \ + python-stdlib/fnmatch \ + python-stdlib/hashlib \ + python-stdlib/inspect \ + python-stdlib/pathlib \ + python-stdlib/quopri \ + python-stdlib/shutil \ + python-stdlib/tempfile \ + python-stdlib/time \ + python-stdlib/unittest/tests \ + python-stdlib/unittest-discover/tests \ + ; do + (cd $path && $MICROPYTHON -m unittest) + if [ $? -ne 0 ]; then false; return; fi + done + + (cd micropython/usb/usb-device && $MICROPYTHON -m tests.test_core_buffer) + if [ $? -ne 0 ]; then false; return; fi + + (cd python-ecosys/cbor2 && $MICROPYTHON -m examples.cbor_test) + if [ $? -ne 0 ]; then false; return; fi } ######################################################################################## @@ -33,10 +124,119 @@ function ci_build_packages_check_manifest { for file in $(find -name manifest.py); do echo "##################################################" echo "# Testing $file" - python3 /tmp/micropython/tools/manifestfile.py --lib . --compile $file + extra_args= + if [[ "$file" =~ "/unix-ffi/" ]]; then + extra_args="--unix-ffi" + fi + python3 /tmp/micropython/tools/manifestfile.py $extra_args --lib . --compile $file done } function ci_build_packages_compile_index { - python3 tools/build.py --micropython /tmp/micropython --output /tmp/micropython-lib-deploy + python3 tools/build.py --micropython /tmp/micropython --output $PACKAGE_INDEX_PATH +} + +function ci_build_packages_examples { + for example in $(find -path \*example\*.py); do + /tmp/micropython/mpy-cross/build/mpy-cross $example + done +} + +function ci_push_package_index { + set -euo pipefail + + # Note: This feature is opt-in, so this function is only run by GitHub + # Actions if the MICROPY_PUBLISH_MIP_INDEX repository variable is set to a + # "truthy" value in the "Secrets and variables" -> "Actions" + # -> "Variables" setting of the GitHub repo. + + PAGES_PATH=/tmp/gh-pages + + if git fetch --depth=1 origin gh-pages; then + git worktree add ${PAGES_PATH} gh-pages + cd ${PAGES_PATH} + NEW_BRANCH=0 + else + echo "Creating gh-pages branch for $GITHUB_REPOSITORY..." + git worktree add --force ${PAGES_PATH} HEAD + cd ${PAGES_PATH} + git switch --orphan gh-pages + NEW_BRANCH=1 + fi + + DEST_PATH=${PAGES_PATH}/mip/${GITHUB_REF_NAME} + if [ -d ${DEST_PATH} ]; then + git rm -r ${DEST_PATH} + fi + mkdir -p ${DEST_PATH} + cd ${DEST_PATH} + + cp -r ${PACKAGE_INDEX_PATH}/* . + + git add . + git_bot_commit "Add CI built packages from commit ${GITHUB_SHA} of ${GITHUB_REF_NAME}" + + if [ "$NEW_BRANCH" -eq 0 ]; then + # A small race condition exists here if another CI job pushes to + # gh-pages at the same time, but this narrows the race to the time + # between these two commands. + git pull --rebase origin gh-pages + fi + git push origin gh-pages + + INDEX_URL="https://${github_repository_owner}.github.io/$(echo%20$%7BGITHUB_REPOSITORY%7D%20|%20cut%20-d'/' -f2-)/mip/${GITHUB_REF_NAME}" + + echo "" + echo "--------------------------------------------------" + echo "Uploaded package files to GitHub Pages." + echo "" + echo "Unless GitHub Pages is disabled on this repo, these files can be installed remotely with:" + echo "" + echo "mpremote mip install --index ${INDEX_URL} PACKAGE_NAME" + echo "" + echo "or on the device as:" + echo "" + echo "import mip" + echo "mip.install(PACKAGE_NAME, index=\"${INDEX_URL}\")" +} + +function ci_cleanup_package_index() +{ + if ! git fetch --depth=1 origin gh-pages; then + exit 0 + fi + + # Argument $1 is github.event.ref, passed in from workflow file. + # + # this value seems to be a REF_NAME, without heads/ or tags/ prefix. (Can't + # use GITHUB_REF_NAME, this evaluates to the default branch.) + DELETED_REF="$1" + + if [ -z "$DELETED_REF" ]; then + echo "Bad DELETE_REF $DELETED_REF" + exit 1 # Internal error with ref format, better than removing all mip/ directory in a commit + fi + + # We need Actions to check out default branch and run tools/ci.sh, but then + # we switch branches + git switch gh-pages + + echo "Removing any published packages for ${DELETED_REF}..." + if [ -d mip/${DELETED_REF} ]; then + git rm -r mip/${DELETED_REF} + git_bot_commit "Remove CI built packages from deleted ${DELETED_REF}" + git pull --rebase origin gh-pages + git push origin gh-pages + else + echo "Nothing to remove." + fi +} + +# Make a git commit with bot authorship +# Argument $1 is the commit message +function git_bot_commit { + # Ref https://github.com/actions/checkout/discussions/479 + git config user.name 'github-actions[bot]' + git config user.email 'github-actions[bot]@users.noreply.github.com' + git commit -m "$1" } diff --git a/tools/codeformat.py b/tools/codeformat.py index 63c9c5988..6a7f2b35f 100755 --- a/tools/codeformat.py +++ b/tools/codeformat.py @@ -5,7 +5,7 @@ # The MIT License (MIT) # # Copyright (c) 2020 Damien P. George -# Copyright (c) 2020 Jim Mussared +# Copyright (c) 2023 Jim Mussared # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,202 +25,19 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -import argparse -import glob -import itertools +# This is just a wrapper around running ruff format, so that code formatting can be +# invoked in the same way as in the main repo. + import os -import re import subprocess -# Relative to top-level repo dir. -PATHS = [ - # C - "**/*.[ch]", - # Python - "**/*.py", -] - -EXCLUSIONS = [] - # Path to repo top-level dir. TOP = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -UNCRUSTIFY_CFG = os.path.join(TOP, "tools/uncrustify.cfg") - -C_EXTS = ( - ".c", - ".h", -) -PY_EXTS = (".py",) - - -MAIN_BRANCH = "master" -BASE_BRANCH = os.environ.get("GITHUB_BASE_REF", MAIN_BRANCH) - - -def list_files(paths, exclusions=None, prefix=""): - files = set() - for pattern in paths: - files.update(glob.glob(os.path.join(prefix, pattern), recursive=True)) - for pattern in exclusions or []: - files.difference_update(glob.fnmatch.filter(files, os.path.join(prefix, pattern))) - return sorted(files) - - -def fixup_c(filename): - # Read file. - with open(filename) as f: - lines = f.readlines() - - # Write out file with fixups. - with open(filename, "w", newline="") as f: - dedent_stack = [] - while lines: - # Get next line. - l = lines.pop(0) - - # Dedent #'s to match indent of following line (not previous line). - m = re.match(r"( +)#(if |ifdef |ifndef |elif |else|endif)", l) - if m: - indent = len(m.group(1)) - directive = m.group(2) - if directive in ("if ", "ifdef ", "ifndef "): - l_next = lines[0] - indent_next = len(re.match(r"( *)", l_next).group(1)) - if indent - 4 == indent_next and re.match(r" +(} else |case )", l_next): - # This #-line (and all associated ones) needs dedenting by 4 spaces. - l = l[4:] - dedent_stack.append(indent - 4) - else: - # This #-line does not need dedenting. - dedent_stack.append(-1) - else: - if dedent_stack[-1] >= 0: - # This associated #-line needs dedenting to match the #if. - indent_diff = indent - dedent_stack[-1] - assert indent_diff >= 0 - l = l[indent_diff:] - if directive == "endif": - dedent_stack.pop() - - # Write out line. - f.write(l) - - assert not dedent_stack, filename - - -def query_git_files(verbose): - def cmd_result_set(cmd): - ret = subprocess.run(cmd, capture_output=True).stdout.strip().decode() - if not ret: - return set() - return {f.strip() for f in ret.split("\n")} - - def rel_paths(files, root): - return {os.path.relpath(os.path.join(root, f.strip()), ".") for f in files} - - try: - ret = set() - - # get path to root of repository - root_dir = ( - subprocess.run(["git", "rev-parse", "--show-toplevel"], capture_output=True) - .stdout.strip() - .decode() - ) - - # Check locally modified files - status = cmd_result_set(["git", "status", "--porcelain"]) - dirty_files = rel_paths({line.split(" ", 1)[-1] for line in status}, root_dir) - ret |= dirty_files - - # Current commit and branch - current_commit = ( - subprocess.run(["git", "rev-parse", "HEAD"], capture_output=True) - .stdout.strip() - .decode() - ) - current_branches = cmd_result_set(["git", "branch", "--contains", current_commit]) - if MAIN_BRANCH in current_branches: - if ret: - if verbose: - print("Local changes detected, only scanning them.") - return ret - - # We're on clean master, run on entire repo - if verbose: - print("Scanning whole repository") - return None - - # List the files modified on current branch - if verbose: - print("Scanning changes from current branch and any local changes") - files_on_branch = rel_paths( - cmd_result_set(["git", "diff", "--name-only", BASE_BRANCH]), root_dir - ) - ret |= files_on_branch - return ret - except: - # Git not available, run on entire repo - return None - def main(): - cmd_parser = argparse.ArgumentParser(description="Auto-format C and Python files.") - cmd_parser.add_argument("-c", action="http://github.com/store_true", help="Format C code only") - cmd_parser.add_argument("-p", action="http://github.com/store_true", help="Format Python code only") - cmd_parser.add_argument("-v", action="http://github.com/store_true", help="Enable verbose output") - cmd_parser.add_argument( - "files", - nargs="*", - help="Run on specific globs. If not specied current branch changes will be used", - ) - args = cmd_parser.parse_args() - - # Setting only one of -c or -p disables the other. If both or neither are set, then do both. - format_c = args.c or not args.p - format_py = args.p or not args.c - - # Expand the globs passed on the command line, or use the default globs above. - files = [] - if args.files: - files = list_files(args.files) - else: - files = query_git_files(verbose=args.v) - if not files: - files = list_files(PATHS, EXCLUSIONS, TOP) - - # Extract files matching a specific language. - def lang_files(exts): - for file in files: - if os.path.splitext(file)[1].lower() in exts: - yield file - - # Run tool on N files at a time (to avoid making the command line too long). - def batch(cmd, files, N=200): - while True: - file_args = list(itertools.islice(files, N)) - if not file_args: - break - subprocess.check_call(cmd + file_args) - - # Format C files with uncrustify. - if format_c: - command = ["uncrustify", "-c", UNCRUSTIFY_CFG, "-lC", "--no-backup"] - if not args.v: - command.append("-q") - batch(command, lang_files(C_EXTS)) - for file in lang_files(C_EXTS): - fixup_c(file) - - # Format Python files with black. - if format_py: - command = ["black", "--fast", "--line-length=99"] - if args.v: - command.append("-v") - else: - command.append("-q") - batch(command, lang_files(PY_EXTS)) + command = ["ruff", "format", "."] + subprocess.check_call(command, cwd=TOP) if __name__ == "__main__": diff --git a/tools/makepyproject.py b/tools/makepyproject.py new file mode 100755 index 000000000..25c05d05f --- /dev/null +++ b/tools/makepyproject.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# +# This file is part of the MicroPython project, http://micropython.org/ +# +# The MIT License (MIT) +# +# Copyright (c) 2023 Jim Mussared +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +# This script makes a CPython-compatible package from a micropython-lib package +# with a pyproject.toml that can be built (via hatch) and deployed to PyPI. +# Requires that the project sets the pypi_publish= kwarg in its metadata(). + +# Usage: +# ./tools/makepyproject.py --output /tmp/foo micropython/foo +# python -m build /tmp/foo +# python -m twine upload /tmp/foo/dist/*.whl + +from email.utils import parseaddr +import os +import re +import shutil +import sys + +from build import error_color, ensure_path_exists + + +DEFAULT_AUTHOR = "micropython-lib " +DEFAULT_LICENSE = "MIT" + + +def quoted_escape(s): + return s.replace('"', '\\"') + + +def build(manifest_path, output_path): + import manifestfile + + if not manifest_path.endswith(".py"): + # Allow specifying either the directory or the manifest file explicitly. + manifest_path = os.path.join(manifest_path, "manifest.py") + + print("Generating pyproject for {} in {}...".format(manifest_path, output_path)) + + toml_path = os.path.join(output_path, "pyproject.toml") + ensure_path_exists(toml_path) + + path_vars = { + "MPY_LIB_DIR": os.path.abspath(os.path.join(os.path.dirname(__file__), "..")), + } + + # .../foo/manifest.py -> foo + package_name = os.path.basename(os.path.dirname(manifest_path)) + + # Compile the manifest. + manifest = manifestfile.ManifestFile(manifestfile.MODE_PYPROJECT, path_vars) + manifest.execute(manifest_path) + + # If a package doesn't have a pypi name, then assume it isn't intended to + # be publishable. + if not manifest.metadata().pypi_publish: + print(error_color("Error:"), package_name, "doesn't have a pypi_publish name.") + sys.exit(1) + + # These should be in all packages eventually. + if not manifest.metadata().version: + print(error_color("Error:"), package_name, "doesn't have a version.") + sys.exit(1) + if not manifest.metadata().description: + print(error_color("Error:"), package_name, "doesn't have a description.") + sys.exit(1) + + # This is the root path of all .py files that are copied. We ensure that + # they all match. + top_level_package = None + + for result in manifest.files(): + # This isn't allowed in micropython-lib anyway. + if result.file_type != manifestfile.FILE_TYPE_LOCAL: + print(error_color("Error:"), "Non-local file not supported.", file=sys.stderr) + sys.exit(1) + + # "foo/bar/baz.py" --> "foo" + # "baz.py" --> "" + result_package = os.path.split(result.target_path)[0] + + if not result_package: + # This is a standalone .py file. + print( + error_color("Error:"), + "Unsupported single-file module: {}".format(result.target_path), + file=sys.stderr, + ) + sys.exit(1) + if top_level_package and result_package != top_level_package: + # This likely suggests that something needs to use require(..., pypi="..."). + print( + error_color("Error:"), + "More than one top-level package: {}, {}.".format( + result_package, top_level_package + ), + file=sys.stderr, + ) + sys.exit(1) + top_level_package = result_package + + # Tag each file with the package metadata and copy the .py directly. + with manifestfile.tagged_py_file(result.full_path, result.metadata) as tagged_path: + dest_path = os.path.join(output_path, result.target_path) + ensure_path_exists(dest_path) + shutil.copyfile(tagged_path, dest_path) + + # Copy README.md if it exists + readme_path = os.path.join(os.path.dirname(manifest_path), "README.md") + readme_toml = "" + if os.path.exists(readme_path): + shutil.copyfile(readme_path, os.path.join(output_path, "README.md")) + readme_toml = 'readme = "README.md"' + + # Apply default author and license, otherwise use the package metadata. + license_toml = 'license = {{ text = "{}" }}'.format( + quoted_escape(manifest.metadata().license or DEFAULT_LICENSE) + ) + author_name, author_email = parseaddr(manifest.metadata().author or DEFAULT_AUTHOR) + author_toml = 'authors = [ {{ name = "{}", email = "{}"}} ]'.format( + quoted_escape(author_name), quoted_escape(author_email) + ) + + # Write pyproject.toml. + with open(toml_path, "w") as toml_file: + print("# Generated by makepyproject.py", file=toml_file) + + print( + """ +[build-system] +requires = [ + "hatchling" +] +build-backend = "hatchling.build" +""", + file=toml_file, + ) + + print( + """ +[project] +name = "{}" +description = "{}" +{} +{} +version = "{}" +dependencies = [{}] +urls = {{ Homepage = "https://github.com/micropython/micropython-lib" }} +{} +""".format( + quoted_escape(manifest.metadata().pypi_publish), + quoted_escape(manifest.metadata().description), + author_toml, + license_toml, + quoted_escape(manifest.metadata().version), + ", ".join('"{}"'.format(quoted_escape(r)) for r in manifest.pypi_dependencies()), + readme_toml, + ), + file=toml_file, + ) + + print( + """ +[tool.hatch.build] +packages = ["{}"] +""".format(top_level_package), + file=toml_file, + ) + + print("Done.") + + +def main(): + import argparse + + cmd_parser = argparse.ArgumentParser( + description="Generate a project that can be pushed to PyPI." + ) + cmd_parser.add_argument("--output", required=True, help="output directory") + cmd_parser.add_argument("--micropython", default=None, help="path to micropython repo") + cmd_parser.add_argument("manifest", help="input package path") + args = cmd_parser.parse_args() + + if args.micropython: + sys.path.append(os.path.join(args.micropython, "tools")) # for manifestfile + + build(args.manifest, args.output) + + +if __name__ == "__main__": + main() diff --git a/tools/verifygitlog.py b/tools/verifygitlog.py new file mode 100755 index 000000000..46fec1e0c --- /dev/null +++ b/tools/verifygitlog.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 + +# This is an exact duplicate of verifygitlog.py from the main repo. + +import re +import subprocess +import sys + +verbosity = 0 # Show what's going on, 0 1 or 2. +suggestions = 1 # Set to 0 to not include lengthy suggestions in error messages. + +ignore_prefixes = [] + + +def verbose(*args): + if verbosity: + print(*args) + + +def very_verbose(*args): + if verbosity > 1: + print(*args) + + +class ErrorCollection: + # Track errors and warnings as the program runs + def __init__(self): + self.has_errors = False + self.has_warnings = False + self.prefix = "" + + def error(self, text): + print("error: {}{}".format(self.prefix, text)) + self.has_errors = True + + def warning(self, text): + print("warning: {}{}".format(self.prefix, text)) + self.has_warnings = True + + +def git_log(pretty_format, *args): + # Delete pretty argument from user args so it doesn't interfere with what we do. + args = ["git", "log"] + [arg for arg in args if "--pretty" not in args] + args.append("--pretty=format:" + pretty_format) + very_verbose("git_log", *args) + # Generator yielding each output line. + for line in subprocess.Popen(args, stdout=subprocess.PIPE).stdout: + yield line.decode().rstrip("\r\n") + + +def diagnose_subject_line(subject_line, subject_line_format, err): + err.error('Subject line: "' + subject_line + '"') + if not subject_line.endswith("."): + err.error('* must end with "."') + if not re.match(r"^[^!]+: ", subject_line): + err.error('* must start with "path: "') + if re.match(r"^[^!]+: *$", subject_line): + err.error("* must contain a subject after the path.") + m = re.match(r"^[^!]+: ([a-z][^ ]*)", subject_line) + if m: + err.error('* first word of subject ("{}") must be capitalised.'.format(m.group(1))) + if re.match(r"^[^!]+: [^ ]+$", subject_line): + err.error("* subject must contain more than one word.") + err.error("* must match: " + repr(subject_line_format)) + err.error('* Example: "py/runtime: Add support for foo to bar."') + + +def verify(sha, err): + verbose("verify", sha) + err.prefix = "commit " + sha + ": " + + # Author and committer email. + for line in git_log("%ae%n%ce", sha, "-n1"): + very_verbose("email", line) + if "noreply" in line: + err.error("Unwanted email address: " + line) + + # Message body. + raw_body = list(git_log("%B", sha, "-n1")) + verify_message_body(raw_body, err) + + +def verify_message_body(raw_body, err): + if not raw_body: + err.error("Message is empty") + return + + # Subject line. + subject_line = raw_body[0] + for prefix in ignore_prefixes: + if subject_line.startswith(prefix): + verbose("Skipping ignored commit message") + return + very_verbose("subject_line", subject_line) + subject_line_format = r"^[^!]+: [A-Z]+.+ .+\.$" + if not re.match(subject_line_format, subject_line): + diagnose_subject_line(subject_line, subject_line_format, err) + if len(subject_line) >= 73: + err.error("Subject line must be 72 or fewer characters: " + subject_line) + + # Do additional checks on the prefix of the subject line. + verify_subject_line_prefix(subject_line.split(": ")[0], err) + + # Second one divides subject and body. + if len(raw_body) > 1 and raw_body[1]: + err.error("Second message line must be empty: " + raw_body[1]) + + # Message body lines. + for line in raw_body[2:]: + # Long lines with URLs or human names are exempt from the line length rule. + if len(line) >= 76 and not ( + "://" in line + or line.startswith("Co-authored-by: ") + or line.startswith("Signed-off-by: ") + ): + err.error("Message lines should be 75 or less characters: " + line) + + if not raw_body[-1].startswith("Signed-off-by: ") or "@" not in raw_body[-1]: + err.error('Message must be signed-off. Use "git commit -s".') + + +def verify_subject_line_prefix(prefix, err): + ext = (".c", ".h", ".cpp", ".js", ".rst", ".md") + + if prefix.startswith((".", "/")): + err.error('Subject prefix cannot begin with "." or "/".') + + if prefix.endswith("/"): + err.error('Subject prefix cannot end with "/".') + + if prefix.startswith("ports/"): + err.error( + 'Subject prefix cannot begin with "ports/", start with the name of the port instead.' + ) + + if prefix.endswith(ext): + err.error( + "Subject prefix cannot end with a file extension, use the main part of the filename without the extension." + ) + + +def run(args): + verbose("run", *args) + + err = ErrorCollection() + + if "--check-file" in args: + filename = args[-1] + verbose("checking commit message from", filename) + with open(args[-1]) as f: + # Remove comment lines as well as any empty lines at the end. + lines = [line.rstrip("\r\n") for line in f if not line.startswith("#")] + while not lines[-1]: + lines.pop() + verify_message_body(lines, err) + else: # Normal operation, pass arguments to git log + for sha in git_log("%h", *args): + verify(sha, err) + + if err.has_errors or err.has_warnings: + if suggestions: + print("See https://github.com/micropython/micropython/blob/master/CODECONVENTIONS.md") + else: + print("ok") + if err.has_errors: + sys.exit(1) + + +def show_help(): + print("usage: verifygitlog.py [-v -n -h --check-file] ...") + print("-v : increase verbosity, can be specified multiple times") + print("-n : do not print multi-line suggestions") + print("-h : print this help message and exit") + print( + "--check-file : Pass a single argument which is a file containing a candidate commit message" + ) + print( + "--ignore-rebase : Skip checking commits with git rebase autosquash prefixes or WIP as a prefix" + ) + print("... : arguments passed to git log to retrieve commits to verify") + print(" see https://www.git-scm.com/docs/git-log") + print(" passing no arguments at all will verify all commits") + print("examples:") + print("verifygitlog.py -n10 # Check last 10 commits") + print("verifygitlog.py -v master..HEAD # Check commits since master") + + +if __name__ == "__main__": + args = sys.argv[1:] + verbosity = args.count("-v") + suggestions = args.count("-n") == 0 + if "--ignore-rebase" in args: + args.remove("--ignore-rebase") + ignore_prefixes = ["squash!", "fixup!", "amend!", "WIP"] + + if "-h" in args: + show_help() + else: + args = [arg for arg in args if arg not in ["-v", "-n", "-h"]] + run(args) diff --git a/unix-ffi/README.md b/unix-ffi/README.md index d6b9417d8..6ea05d65a 100644 --- a/unix-ffi/README.md +++ b/unix-ffi/README.md @@ -19,9 +19,13 @@ replacement for CPython. ### Usage -To use a unix-specific library, pass `unix_ffi=True` to `require()` in your -manifest file. +To use a unix-specific library, a manifest file must add the `unix-ffi` +library to the library search path using `add_library()`: ```py -require("os", unix_ffi=True) # Use the unix-ffi version instead of python-stdlib. +add_library("unix-ffi", "$(MPY_LIB_DIR)/unix-ffi", prepend=True) ``` + +Prepending the `unix-ffi` library to the path will make it so that the +`unix-ffi` version of a package will be preferred if that package appears in +both `unix-ffi` and another library (eg `python-stdlib`). diff --git a/unix-ffi/_markupbase/manifest.py b/unix-ffi/_markupbase/manifest.py index 983b57995..9cbf52bb0 100644 --- a/unix-ffi/_markupbase/manifest.py +++ b/unix-ffi/_markupbase/manifest.py @@ -1,5 +1,5 @@ -metadata(version="3.3.3-1") +metadata(version="3.3.4") -require("re", unix_ffi=True) +require("re") module("_markupbase.py") diff --git a/unix-ffi/cgi/cgi.py b/unix-ffi/cgi/cgi.py index 79cc4a738..550f70713 100644 --- a/unix-ffi/cgi/cgi.py +++ b/unix-ffi/cgi/cgi.py @@ -25,9 +25,6 @@ # responsible for its maintenance. # -__version__ = "2.6" - - # Imports # ======= diff --git a/unix-ffi/cgi/manifest.py b/unix-ffi/cgi/manifest.py index b1db6ce30..29732c939 100644 --- a/unix-ffi/cgi/manifest.py +++ b/unix-ffi/cgi/manifest.py @@ -1,3 +1,3 @@ -metadata(version="3.3.3-2") +metadata(version="3.3.4") module("cgi.py") diff --git a/unix-ffi/email.charset/manifest.py b/unix-ffi/email.charset/manifest.py index 31d70cece..7e6dd7936 100644 --- a/unix-ffi/email.charset/manifest.py +++ b/unix-ffi/email.charset/manifest.py @@ -1,7 +1,7 @@ metadata(version="0.5.1") require("functools") -require("email.encoders", unix_ffi=True) -require("email.errors", unix_ffi=True) +require("email.encoders") +require("email.errors") package("email") diff --git a/unix-ffi/email.encoders/manifest.py b/unix-ffi/email.encoders/manifest.py index e1e2090c9..a3e735d8c 100644 --- a/unix-ffi/email.encoders/manifest.py +++ b/unix-ffi/email.encoders/manifest.py @@ -3,7 +3,7 @@ require("base64") require("binascii") require("quopri") -require("re", unix_ffi=True) +require("re") require("string") package("email") diff --git a/unix-ffi/email.feedparser/manifest.py b/unix-ffi/email.feedparser/manifest.py index 4ea80e302..31c34ba91 100644 --- a/unix-ffi/email.feedparser/manifest.py +++ b/unix-ffi/email.feedparser/manifest.py @@ -1,8 +1,8 @@ metadata(version="0.5.1") -require("re", unix_ffi=True) -require("email.errors", unix_ffi=True) -require("email.message", unix_ffi=True) -require("email.internal", unix_ffi=True) +require("re") +require("email.errors") +require("email.message") +require("email.internal") package("email") diff --git a/unix-ffi/email.header/manifest.py b/unix-ffi/email.header/manifest.py index 65b017b50..0be7e85c2 100644 --- a/unix-ffi/email.header/manifest.py +++ b/unix-ffi/email.header/manifest.py @@ -1,9 +1,9 @@ metadata(version="0.5.2") -require("re", unix_ffi=True) +require("re") require("binascii") -require("email.encoders", unix_ffi=True) -require("email.errors", unix_ffi=True) -require("email.charset", unix_ffi=True) +require("email.encoders") +require("email.errors") +require("email.charset") package("email") diff --git a/unix-ffi/email.internal/manifest.py b/unix-ffi/email.internal/manifest.py index 4aff6d2c5..88acb2c01 100644 --- a/unix-ffi/email.internal/manifest.py +++ b/unix-ffi/email.internal/manifest.py @@ -1,15 +1,15 @@ metadata(version="0.5.1") -require("re", unix_ffi=True) +require("re") require("base64") require("binascii") require("functools") require("string") # require("calendar") TODO require("abc") -require("email.errors", unix_ffi=True) -require("email.header", unix_ffi=True) -require("email.charset", unix_ffi=True) -require("email.utils", unix_ffi=True) +require("email.errors") +require("email.header") +require("email.charset") +require("email.utils") package("email") diff --git a/unix-ffi/email.message/manifest.py b/unix-ffi/email.message/manifest.py index 7b75ee7ac..d1849de35 100644 --- a/unix-ffi/email.message/manifest.py +++ b/unix-ffi/email.message/manifest.py @@ -1,11 +1,11 @@ metadata(version="0.5.3") -require("re", unix_ffi=True) +require("re") require("uu") require("base64") require("binascii") -require("email.utils", unix_ffi=True) -require("email.errors", unix_ffi=True) -require("email.charset", unix_ffi=True) +require("email.utils") +require("email.errors") +require("email.charset") package("email") diff --git a/unix-ffi/email.parser/manifest.py b/unix-ffi/email.parser/manifest.py index ebe662111..dd8aacde8 100644 --- a/unix-ffi/email.parser/manifest.py +++ b/unix-ffi/email.parser/manifest.py @@ -1,8 +1,8 @@ metadata(version="0.5.1") require("warnings") -require("email.feedparser", unix_ffi=True) -require("email.message", unix_ffi=True) -require("email.internal", unix_ffi=True) +require("email.feedparser") +require("email.message") +require("email.internal") package("email") diff --git a/unix-ffi/email.utils/manifest.py b/unix-ffi/email.utils/manifest.py index be6e33183..a7208536d 100644 --- a/unix-ffi/email.utils/manifest.py +++ b/unix-ffi/email.utils/manifest.py @@ -1,13 +1,13 @@ -metadata(version="3.3.3-2") +metadata(version="3.3.4") -require("os", unix_ffi=True) -require("re", unix_ffi=True) +require("os") +require("re") require("base64") require("random") require("datetime") -require("urllib.parse", unix_ffi=True) +require("urllib.parse") require("warnings") require("quopri") -require("email.charset", unix_ffi=True) +require("email.charset") package("email") diff --git a/unix-ffi/fcntl/manifest.py b/unix-ffi/fcntl/manifest.py index a0e9d9592..e572a58e8 100644 --- a/unix-ffi/fcntl/manifest.py +++ b/unix-ffi/fcntl/manifest.py @@ -2,6 +2,6 @@ # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) +require("ffilib") module("fcntl.py") diff --git a/unix-ffi/getopt/manifest.py b/unix-ffi/getopt/manifest.py index 2038e7504..cde6c4c09 100644 --- a/unix-ffi/getopt/manifest.py +++ b/unix-ffi/getopt/manifest.py @@ -1,5 +1,5 @@ -metadata(version="3.3.3-1") +metadata(version="3.3.4") -require("os", unix_ffi=True) +require("os") module("getopt.py") diff --git a/unix-ffi/gettext/manifest.py b/unix-ffi/gettext/manifest.py index 527330e92..fe40b01b6 100644 --- a/unix-ffi/gettext/manifest.py +++ b/unix-ffi/gettext/manifest.py @@ -1,7 +1,7 @@ -metadata(version="0.1") +metadata(version="0.1.0") # Originally written by Riccardo Magliocchetti. -require("ffilib", unix_ffi=True) +require("ffilib") module("gettext.py") diff --git a/unix-ffi/glob/manifest.py b/unix-ffi/glob/manifest.py index 622289bca..2d2fab31c 100644 --- a/unix-ffi/glob/manifest.py +++ b/unix-ffi/glob/manifest.py @@ -1,8 +1,8 @@ metadata(version="0.5.2") -require("os", unix_ffi=True) +require("os") require("os-path") -require("re", unix_ffi=True) +require("re") require("fnmatch") module("glob.py") diff --git a/unix-ffi/html.entities/manifest.py b/unix-ffi/html.entities/manifest.py index 0a612905f..a985d2821 100644 --- a/unix-ffi/html.entities/manifest.py +++ b/unix-ffi/html.entities/manifest.py @@ -1,3 +1,3 @@ -metadata(version="3.3.3-1") +metadata(version="3.3.4") package("html") diff --git a/unix-ffi/html.parser/manifest.py b/unix-ffi/html.parser/manifest.py index 9c82a7833..3f29bbceb 100644 --- a/unix-ffi/html.parser/manifest.py +++ b/unix-ffi/html.parser/manifest.py @@ -1,8 +1,8 @@ -metadata(version="3.3.3-2") +metadata(version="3.3.4") -require("_markupbase", unix_ffi=True) +require("_markupbase") require("warnings") -require("html.entities", unix_ffi=True) -require("re", unix_ffi=True) +require("html.entities") +require("re") package("html") diff --git a/unix-ffi/http.client/manifest.py b/unix-ffi/http.client/manifest.py index be0c9ef36..add274422 100644 --- a/unix-ffi/http.client/manifest.py +++ b/unix-ffi/http.client/manifest.py @@ -1,10 +1,10 @@ metadata(version="0.5.1") -require("email.parser", unix_ffi=True) -require("email.message", unix_ffi=True) -require("socket", unix_ffi=True) +require("email.parser") +require("email.message") +require("socket") require("collections") -require("urllib.parse", unix_ffi=True) +require("urllib.parse") require("warnings") package("http") diff --git a/python-stdlib/json/json/__init__.py b/unix-ffi/json/json/__init__.py similarity index 98% rename from python-stdlib/json/json/__init__.py rename to unix-ffi/json/json/__init__.py index eb9493edb..954618f33 100644 --- a/python-stdlib/json/json/__init__.py +++ b/unix-ffi/json/json/__init__.py @@ -99,7 +99,7 @@ $ echo '{ 1.2:3.4}' | python -m json.tool Expecting property name enclosed in double quotes: line 1 column 3 (char 2) """ -__version__ = "2.0.9" + __all__ = [ "dump", "dumps", @@ -354,8 +354,8 @@ def loads( object_pairs_hook=None, **kw ): - """Deserialize ``s`` (a ``str`` instance containing a JSON - document) to a Python object. + """Deserialize ``s`` (a ``str``, ``bytes`` or ``bytearray`` instance + containing a JSON document) to a Python object. ``object_hook`` is an optional function that will be called with the result of any object literal decode (a ``dict``). The return value of @@ -413,4 +413,6 @@ def loads( kw["parse_int"] = parse_int if parse_constant is not None: kw["parse_constant"] = parse_constant + if isinstance(s, (bytes, bytearray)): + s = s.decode('utf-8') return cls(**kw).decode(s) diff --git a/python-stdlib/json/json/decoder.py b/unix-ffi/json/json/decoder.py similarity index 100% rename from python-stdlib/json/json/decoder.py rename to unix-ffi/json/json/decoder.py diff --git a/python-stdlib/json/json/encoder.py b/unix-ffi/json/json/encoder.py similarity index 100% rename from python-stdlib/json/json/encoder.py rename to unix-ffi/json/json/encoder.py diff --git a/python-stdlib/json/json/scanner.py b/unix-ffi/json/json/scanner.py similarity index 100% rename from python-stdlib/json/json/scanner.py rename to unix-ffi/json/json/scanner.py diff --git a/python-stdlib/json/json/tool.py b/unix-ffi/json/json/tool.py similarity index 100% rename from python-stdlib/json/json/tool.py rename to unix-ffi/json/json/tool.py diff --git a/unix-ffi/json/manifest.py b/unix-ffi/json/manifest.py new file mode 100644 index 000000000..9267719f1 --- /dev/null +++ b/unix-ffi/json/manifest.py @@ -0,0 +1,4 @@ +metadata(version="0.2.0") + +require("re") +package("json") diff --git a/python-stdlib/json/test_json.py b/unix-ffi/json/test_json.py similarity index 100% rename from python-stdlib/json/test_json.py rename to unix-ffi/json/test_json.py diff --git a/unix-ffi/machine/example_timer.py b/unix-ffi/machine/example_timer.py index a0d44110f..550d68cd3 100644 --- a/unix-ffi/machine/example_timer.py +++ b/unix-ffi/machine/example_timer.py @@ -1,4 +1,4 @@ -import utime +import time from machine import Timer @@ -7,5 +7,5 @@ t1.callback(lambda t: print(t, "tick1")) t2.callback(lambda t: print(t, "tick2")) -utime.sleep(3) +time.sleep(3) print("done") diff --git a/unix-ffi/machine/machine/timer.py b/unix-ffi/machine/machine/timer.py index 1aa53f936..be00cee33 100644 --- a/unix-ffi/machine/machine/timer.py +++ b/unix-ffi/machine/machine/timer.py @@ -1,13 +1,14 @@ import ffilib import uctypes import array -import uos import os -import utime from signal import * libc = ffilib.libc() -librt = ffilib.open("librt") +try: + librt = ffilib.open("librt") +except OSError as e: + librt = libc CLOCK_REALTIME = 0 CLOCK_MONOTONIC = 1 diff --git a/unix-ffi/machine/manifest.py b/unix-ffi/machine/manifest.py index 9c1f34775..f7c11b81a 100644 --- a/unix-ffi/machine/manifest.py +++ b/unix-ffi/machine/manifest.py @@ -1,9 +1,9 @@ -metadata(version="0.2.1") +metadata(version="0.2.2") # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) -require("os", unix_ffi=True) -require("signal", unix_ffi=True) +require("ffilib") +require("os") +require("signal") package("machine") diff --git a/unix-ffi/multiprocessing/manifest.py b/unix-ffi/multiprocessing/manifest.py index 68f2bca08..d6b32411d 100644 --- a/unix-ffi/multiprocessing/manifest.py +++ b/unix-ffi/multiprocessing/manifest.py @@ -2,8 +2,8 @@ # Originally written by Paul Sokolovsky. -require("os", unix_ffi=True) -require("select", unix_ffi=True) +require("os") +require("select") require("pickle") module("multiprocessing.py") diff --git a/unix-ffi/os/manifest.py b/unix-ffi/os/manifest.py index 38cb87d5a..e4bc100a2 100644 --- a/unix-ffi/os/manifest.py +++ b/unix-ffi/os/manifest.py @@ -1,8 +1,8 @@ -metadata(version="0.6") +metadata(version="0.6.0") # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) +require("ffilib") require("errno") require("stat") diff --git a/unix-ffi/os/os/__init__.py b/unix-ffi/os/os/__init__.py index 3cca078f9..6c87da892 100644 --- a/unix-ffi/os/os/__init__.py +++ b/unix-ffi/os/os/__init__.py @@ -1,5 +1,5 @@ import array -import ustruct as struct +import struct import errno as errno_ import stat as stat_ import ffilib diff --git a/unix-ffi/pwd/manifest.py b/unix-ffi/pwd/manifest.py index 7db3213f5..fd422aaeb 100644 --- a/unix-ffi/pwd/manifest.py +++ b/unix-ffi/pwd/manifest.py @@ -1,7 +1,7 @@ -metadata(version="0.1") +metadata(version="0.1.0") # Originally written by Riccardo Magliocchetti. -require("ffilib", unix_ffi=True) +require("ffilib") module("pwd.py") diff --git a/unix-ffi/pwd/pwd.py b/unix-ffi/pwd/pwd.py index 29ebe3416..561269ed2 100644 --- a/unix-ffi/pwd/pwd.py +++ b/unix-ffi/pwd/pwd.py @@ -1,8 +1,8 @@ import ffilib import uctypes -import ustruct +import struct -from ucollections import namedtuple +from collections import namedtuple libc = ffilib.libc() @@ -20,6 +20,6 @@ def getpwnam(user): if not passwd: raise KeyError("getpwnam(): name not found: {}".format(user)) passwd_fmt = "SSIISSS" - passwd = uctypes.bytes_at(passwd, ustruct.calcsize(passwd_fmt)) - passwd = ustruct.unpack(passwd_fmt, passwd) + passwd = uctypes.bytes_at(passwd, struct.calcsize(passwd_fmt)) + passwd = struct.unpack(passwd_fmt, passwd) return struct_passwd(*passwd) diff --git a/unix-ffi/pyusb/examples/lsusb.py b/unix-ffi/pyusb/examples/lsusb.py new file mode 100644 index 000000000..549043567 --- /dev/null +++ b/unix-ffi/pyusb/examples/lsusb.py @@ -0,0 +1,18 @@ +# Simple example to list attached USB devices. + +import usb.core + +for device in usb.core.find(find_all=True): + print("ID {:04x}:{:04x}".format(device.idVendor, device.idProduct)) + for cfg in device: + print( + " config numitf={} value={} attr={} power={}".format( + cfg.bNumInterfaces, cfg.bConfigurationValue, cfg.bmAttributes, cfg.bMaxPower + ) + ) + for itf in cfg: + print( + " interface class={} subclass={}".format( + itf.bInterfaceClass, itf.bInterfaceSubClass + ) + ) diff --git a/unix-ffi/pyusb/manifest.py b/unix-ffi/pyusb/manifest.py new file mode 100644 index 000000000..d60076255 --- /dev/null +++ b/unix-ffi/pyusb/manifest.py @@ -0,0 +1,3 @@ +metadata(version="0.1.0", pypi="pyusb") + +package("usb") diff --git a/unix-ffi/pyusb/usb/__init__.py b/unix-ffi/pyusb/usb/__init__.py new file mode 100644 index 000000000..19afe623c --- /dev/null +++ b/unix-ffi/pyusb/usb/__init__.py @@ -0,0 +1,2 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George diff --git a/unix-ffi/pyusb/usb/control.py b/unix-ffi/pyusb/usb/control.py new file mode 100644 index 000000000..b03a89464 --- /dev/null +++ b/unix-ffi/pyusb/usb/control.py @@ -0,0 +1,10 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George + + +def get_descriptor(dev, desc_size, desc_type, desc_index, wIndex=0): + wValue = desc_index | desc_type << 8 + d = dev.ctrl_transfer(0x80, 0x06, wValue, wIndex, desc_size) + if len(d) < 2: + raise Exception("invalid descriptor") + return d diff --git a/unix-ffi/pyusb/usb/core.py b/unix-ffi/pyusb/usb/core.py new file mode 100644 index 000000000..bfb0a028d --- /dev/null +++ b/unix-ffi/pyusb/usb/core.py @@ -0,0 +1,239 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George + +import sys +import ffi +import uctypes + +if sys.maxsize >> 32: + UINTPTR_SIZE = 8 + UINTPTR = uctypes.UINT64 +else: + UINTPTR_SIZE = 4 + UINTPTR = uctypes.UINT32 + + +def _align_word(x): + return (x + UINTPTR_SIZE - 1) & ~(UINTPTR_SIZE - 1) + + +ptr_descriptor = (0 | uctypes.ARRAY, 1 | UINTPTR) + +libusb_device_descriptor = { + "bLength": 0 | uctypes.UINT8, + "bDescriptorType": 1 | uctypes.UINT8, + "bcdUSB": 2 | uctypes.UINT16, + "bDeviceClass": 4 | uctypes.UINT8, + "bDeviceSubClass": 5 | uctypes.UINT8, + "bDeviceProtocol": 6 | uctypes.UINT8, + "bMaxPacketSize0": 7 | uctypes.UINT8, + "idVendor": 8 | uctypes.UINT16, + "idProduct": 10 | uctypes.UINT16, + "bcdDevice": 12 | uctypes.UINT16, + "iManufacturer": 14 | uctypes.UINT8, + "iProduct": 15 | uctypes.UINT8, + "iSerialNumber": 16 | uctypes.UINT8, + "bNumConfigurations": 17 | uctypes.UINT8, +} + +libusb_config_descriptor = { + "bLength": 0 | uctypes.UINT8, + "bDescriptorType": 1 | uctypes.UINT8, + "wTotalLength": 2 | uctypes.UINT16, + "bNumInterfaces": 4 | uctypes.UINT8, + "bConfigurationValue": 5 | uctypes.UINT8, + "iConfiguration": 6 | uctypes.UINT8, + "bmAttributes": 7 | uctypes.UINT8, + "MaxPower": 8 | uctypes.UINT8, + "interface": _align_word(9) | UINTPTR, # array of libusb_interface + "extra": (_align_word(9) + UINTPTR_SIZE) | UINTPTR, + "extra_length": (_align_word(9) + 2 * UINTPTR_SIZE) | uctypes.INT, +} + +libusb_interface = { + "altsetting": 0 | UINTPTR, # array of libusb_interface_descriptor + "num_altsetting": UINTPTR_SIZE | uctypes.INT, +} + +libusb_interface_descriptor = { + "bLength": 0 | uctypes.UINT8, + "bDescriptorType": 1 | uctypes.UINT8, + "bInterfaceNumber": 2 | uctypes.UINT8, + "bAlternateSetting": 3 | uctypes.UINT8, + "bNumEndpoints": 4 | uctypes.UINT8, + "bInterfaceClass": 5 | uctypes.UINT8, + "bInterfaceSubClass": 6 | uctypes.UINT8, + "bInterfaceProtocol": 7 | uctypes.UINT8, + "iInterface": 8 | uctypes.UINT8, + "endpoint": _align_word(9) | UINTPTR, + "extra": (_align_word(9) + UINTPTR_SIZE) | UINTPTR, + "extra_length": (_align_word(9) + 2 * UINTPTR_SIZE) | uctypes.INT, +} + +libusb = ffi.open("libusb-1.0.so") +libusb_init = libusb.func("i", "libusb_init", "p") +libusb_exit = libusb.func("v", "libusb_exit", "p") +libusb_get_device_list = libusb.func("i", "libusb_get_device_list", "pp") # return is ssize_t +libusb_free_device_list = libusb.func("v", "libusb_free_device_list", "pi") +libusb_get_device_descriptor = libusb.func("i", "libusb_get_device_descriptor", "pp") +libusb_get_config_descriptor = libusb.func("i", "libusb_get_config_descriptor", "pBp") +libusb_free_config_descriptor = libusb.func("v", "libusb_free_config_descriptor", "p") +libusb_open = libusb.func("i", "libusb_open", "pp") +libusb_set_configuration = libusb.func("i", "libusb_set_configuration", "pi") +libusb_claim_interface = libusb.func("i", "libusb_claim_interface", "pi") +libusb_control_transfer = libusb.func("i", "libusb_control_transfer", "pBBHHpHI") + + +def _new(sdesc): + buf = bytearray(uctypes.sizeof(sdesc)) + s = uctypes.struct(uctypes.addressof(buf), sdesc) + return s + + +class Interface: + def __init__(self, descr): + # Public attributes. + self.bInterfaceClass = descr.bInterfaceClass + self.bInterfaceSubClass = descr.bInterfaceSubClass + self.iInterface = descr.iInterface + self.extra_descriptors = uctypes.bytes_at(descr.extra, descr.extra_length) + + +class Configuration: + def __init__(self, dev, cfg_idx): + cfgs = _new(ptr_descriptor) + if libusb_get_config_descriptor(dev._dev, cfg_idx, cfgs) != 0: + libusb_exit(0) + raise Exception + descr = uctypes.struct(cfgs[0], libusb_config_descriptor) + + # Extract all needed info because descr is going to be free'd at the end. + self._itfs = [] + itf_array = uctypes.struct( + descr.interface, (0 | uctypes.ARRAY, descr.bNumInterfaces, libusb_interface) + ) + for i in range(descr.bNumInterfaces): + itf = itf_array[i] + alt_array = uctypes.struct( + itf.altsetting, + (0 | uctypes.ARRAY, itf.num_altsetting, libusb_interface_descriptor), + ) + for j in range(itf.num_altsetting): + alt = alt_array[j] + self._itfs.append(Interface(alt)) + + # Public attributes. + self.bNumInterfaces = descr.bNumInterfaces + self.bConfigurationValue = descr.bConfigurationValue + self.bmAttributes = descr.bmAttributes + self.bMaxPower = descr.MaxPower + self.extra_descriptors = uctypes.bytes_at(descr.extra, descr.extra_length) + + # Free descr memory in the driver. + libusb_free_config_descriptor(cfgs[0]) + + def __iter__(self): + return iter(self._itfs) + + +class Device: + _TIMEOUT_DEFAULT = 1000 + + def __init__(self, dev, descr): + self._dev = dev + self._num_cfg = descr.bNumConfigurations + self._handle = None + self._claim_itf = set() + + # Public attributes. + self.idVendor = descr.idVendor + self.idProduct = descr.idProduct + + def __iter__(self): + for i in range(self._num_cfg): + yield Configuration(self, i) + + def __getitem__(self, i): + return Configuration(self, i) + + def _open(self): + if self._handle is None: + # Open the USB device. + handle = _new(ptr_descriptor) + if libusb_open(self._dev, handle) != 0: + libusb_exit(0) + raise Exception + self._handle = handle[0] + + def _claim_interface(self, i): + if libusb_claim_interface(self._handle, i) != 0: + libusb_exit(0) + raise Exception + + def set_configuration(self): + # Select default configuration. + self._open() + cfg = Configuration(self, 0).bConfigurationValue + ret = libusb_set_configuration(self._handle, cfg) + if ret != 0: + libusb_exit(0) + raise Exception + + def ctrl_transfer( + self, bmRequestType, bRequest, wValue=0, wIndex=0, data_or_wLength=None, timeout=None + ): + if data_or_wLength is None: + l = 0 + data = bytes() + elif isinstance(data_or_wLength, int): + l = data_or_wLength + data = bytearray(l) + else: + l = len(data_or_wLength) + data = data_or_wLength + self._open() + if wIndex & 0xFF not in self._claim_itf: + self._claim_interface(wIndex & 0xFF) + self._claim_itf.add(wIndex & 0xFF) + if timeout is None: + timeout = self._TIMEOUT_DEFAULT + ret = libusb_control_transfer( + self._handle, bmRequestType, bRequest, wValue, wIndex, data, l, timeout * 1000 + ) + if ret < 0: + libusb_exit(0) + raise Exception + if isinstance(data_or_wLength, int): + return data + else: + return ret + + +def find(*, find_all=False, custom_match=None, idVendor=None, idProduct=None): + if libusb_init(0) < 0: + raise Exception + + devs = _new(ptr_descriptor) + count = libusb_get_device_list(0, devs) + if count < 0: + libusb_exit(0) + raise Exception + + dev_array = uctypes.struct(devs[0], (0 | uctypes.ARRAY, count | UINTPTR)) + descr = _new(libusb_device_descriptor) + devices = None + for i in range(count): + libusb_get_device_descriptor(dev_array[i], descr) + if idVendor and descr.idVendor != idVendor: + continue + if idProduct and descr.idProduct != idProduct: + continue + device = Device(dev_array[i], descr) + if custom_match and not custom_match(device): + continue + if not find_all: + return device + if not devices: + devices = [] + devices.append(device) + return devices diff --git a/unix-ffi/pyusb/usb/util.py b/unix-ffi/pyusb/usb/util.py new file mode 100644 index 000000000..04e4763e4 --- /dev/null +++ b/unix-ffi/pyusb/usb/util.py @@ -0,0 +1,16 @@ +# MicroPython USB host library, compatible with PyUSB. +# MIT license; Copyright (c) 2021-2024 Damien P. George + +import usb.control + + +def claim_interface(device, interface): + device._claim_interface(interface) + + +def get_string(device, index): + bs = usb.control.get_descriptor(device, 254, 3, index, 0) + s = "" + for i in range(2, bs[0] & 0xFE, 2): + s += chr(bs[i] | bs[i + 1] << 8) + return s diff --git a/unix-ffi/re/manifest.py b/unix-ffi/re/manifest.py index cc52df47a..ca027317d 100644 --- a/unix-ffi/re/manifest.py +++ b/unix-ffi/re/manifest.py @@ -2,6 +2,6 @@ # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) +require("ffilib") module("re.py") diff --git a/unix-ffi/re/re.py b/unix-ffi/re/re.py index d37584320..bd9566cb9 100644 --- a/unix-ffi/re/re.py +++ b/unix-ffi/re/re.py @@ -1,36 +1,55 @@ import sys import ffilib import array +import uctypes +pcre2 = ffilib.open("libpcre2-8") -pcre = ffilib.open("libpcre") +# pcre2_code *pcre2_compile(PCRE2_SPTR pattern, PCRE2_SIZE length, +# uint32_t options, int *errorcode, PCRE2_SIZE *erroroffset, +# pcre2_compile_context *ccontext); +pcre2_compile = pcre2.func("p", "pcre2_compile_8", "siippp") -# pcre *pcre_compile(const char *pattern, int options, -# const char **errptr, int *erroffset, -# const unsigned char *tableptr); -pcre_compile = pcre.func("p", "pcre_compile", "sipps") +# int pcre2_match(const pcre2_code *code, PCRE2_SPTR subject, +# PCRE2_SIZE length, PCRE2_SIZE startoffset, uint32_t options, +# pcre2_match_data *match_data, pcre2_match_context *mcontext); +pcre2_match = pcre2.func("i", "pcre2_match_8", "Psiiipp") -# int pcre_exec(const pcre *code, const pcre_extra *extra, -# const char *subject, int length, int startoffset, -# int options, int *ovector, int ovecsize); -pcre_exec = pcre.func("i", "pcre_exec", "PPsiiipi") +# int pcre2_pattern_info(const pcre2_code *code, uint32_t what, +# void *where); +pcre2_pattern_info = pcre2.func("i", "pcre2_pattern_info_8", "Pip") -# int pcre_fullinfo(const pcre *code, const pcre_extra *extra, -# int what, void *where); -pcre_fullinfo = pcre.func("i", "pcre_fullinfo", "PPip") +# PCRE2_SIZE *pcre2_get_ovector_pointer(pcre2_match_data *match_data); +pcre2_get_ovector_pointer = pcre2.func("p", "pcre2_get_ovector_pointer_8", "p") +# pcre2_match_data *pcre2_match_data_create_from_pattern(const pcre2_code *code, +# pcre2_general_context *gcontext); +pcre2_match_data_create_from_pattern = pcre2.func( + "p", "pcre2_match_data_create_from_pattern_8", "Pp" +) -IGNORECASE = I = 1 -MULTILINE = M = 2 -DOTALL = S = 4 -VERBOSE = X = 8 -PCRE_ANCHORED = 0x10 +# PCRE2_SIZE that is of type size_t. +# Use ULONG as type to support both 32bit and 64bit. +PCRE2_SIZE_SIZE = uctypes.sizeof({"field": 0 | uctypes.ULONG}) +PCRE2_SIZE_TYPE = "L" + +# Real value in pcre2.h is 0xFFFFFFFF for 32bit and +# 0x0xFFFFFFFFFFFFFFFF for 64bit that is equivalent +# to -1 +PCRE2_ZERO_TERMINATED = -1 + + +IGNORECASE = I = 0x8 +MULTILINE = M = 0x400 +DOTALL = S = 0x20 +VERBOSE = X = 0x80 +PCRE2_ANCHORED = 0x80000000 # TODO. Note that Python3 has unicode by default ASCII = A = 0 UNICODE = U = 0 -PCRE_INFO_CAPTURECOUNT = 2 +PCRE2_INFO_CAPTURECOUNT = 0x4 class PCREMatch: @@ -67,19 +86,23 @@ def __init__(self, compiled_ptn): def search(self, s, pos=0, endpos=-1, _flags=0): assert endpos == -1, "pos: %d, endpos: %d" % (pos, endpos) buf = array.array("i", [0]) - pcre_fullinfo(self.obj, None, PCRE_INFO_CAPTURECOUNT, buf) + pcre2_pattern_info(self.obj, PCRE2_INFO_CAPTURECOUNT, buf) cap_count = buf[0] - ov = array.array("i", [0, 0, 0] * (cap_count + 1)) - num = pcre_exec(self.obj, None, s, len(s), pos, _flags, ov, len(ov)) + match_data = pcre2_match_data_create_from_pattern(self.obj, None) + num = pcre2_match(self.obj, s, len(s), pos, _flags, match_data, None) if num == -1: # No match return None + ov_ptr = pcre2_get_ovector_pointer(match_data) + # pcre2_get_ovector_pointer return PCRE2_SIZE + ov_buf = uctypes.bytearray_at(ov_ptr, PCRE2_SIZE_SIZE * (cap_count + 1) * 2) + ov = array.array(PCRE2_SIZE_TYPE, ov_buf) # We don't care how many matching subexpressions we got, we # care only about total # of capturing ones (including empty) return PCREMatch(s, cap_count + 1, ov) def match(self, s, pos=0, endpos=-1): - return self.search(s, pos, endpos, PCRE_ANCHORED) + return self.search(s, pos, endpos, PCRE2_ANCHORED) def sub(self, repl, s, count=0): if not callable(repl): @@ -141,9 +164,9 @@ def findall(self, s): def compile(pattern, flags=0): - errptr = bytes(4) + errcode = bytes(4) erroffset = bytes(4) - regex = pcre_compile(pattern, flags, errptr, erroffset, None) + regex = pcre2_compile(pattern, PCRE2_ZERO_TERMINATED, flags, errcode, erroffset, None) assert regex return PCREPattern(regex) @@ -154,7 +177,7 @@ def search(pattern, string, flags=0): def match(pattern, string, flags=0): - r = compile(pattern, flags | PCRE_ANCHORED) + r = compile(pattern, flags | PCRE2_ANCHORED) return r.search(string) diff --git a/unix-ffi/select/manifest.py b/unix-ffi/select/manifest.py index cadfd4e96..b9576de5e 100644 --- a/unix-ffi/select/manifest.py +++ b/unix-ffi/select/manifest.py @@ -1,8 +1,8 @@ -metadata(version="0.3") +metadata(version="0.3.0") # Originally written by Paul Sokolovsky. -require("os", unix_ffi=True) -require("ffilib", unix_ffi=True) +require("os") +require("ffilib") module("select.py") diff --git a/unix-ffi/select/select.py b/unix-ffi/select/select.py index eec9bfb81..9d514a31d 100644 --- a/unix-ffi/select/select.py +++ b/unix-ffi/select/select.py @@ -1,5 +1,5 @@ import ffi -import ustruct as struct +import struct import os import errno import ffilib diff --git a/unix-ffi/signal/manifest.py b/unix-ffi/signal/manifest.py index 913bbdc8c..cb23542cc 100644 --- a/unix-ffi/signal/manifest.py +++ b/unix-ffi/signal/manifest.py @@ -2,6 +2,6 @@ # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) +require("ffilib") module("signal.py") diff --git a/unix-ffi/sqlite3/manifest.py b/unix-ffi/sqlite3/manifest.py index e941e1ddd..5b04d71d3 100644 --- a/unix-ffi/sqlite3/manifest.py +++ b/unix-ffi/sqlite3/manifest.py @@ -1,7 +1,7 @@ -metadata(version="0.2.4") +metadata(version="0.3.0") # Originally written by Paul Sokolovsky. -require("ffilib", unix_ffi=True) +require("ffilib") module("sqlite3.py") diff --git a/unix-ffi/sqlite3/sqlite3.py b/unix-ffi/sqlite3/sqlite3.py index 0f00ff508..299f8247d 100644 --- a/unix-ffi/sqlite3/sqlite3.py +++ b/unix-ffi/sqlite3/sqlite3.py @@ -1,12 +1,21 @@ import sys import ffilib +import uctypes sq3 = ffilib.open("libsqlite3") +# int sqlite3_open( +# const char *filename, /* Database filename (UTF-8) */ +# sqlite3 **ppDb /* OUT: SQLite db handle */ +# ); sqlite3_open = sq3.func("i", "sqlite3_open", "sp") -# int sqlite3_close(sqlite3*); -sqlite3_close = sq3.func("i", "sqlite3_close", "p") +# int sqlite3_config(int, ...); +sqlite3_config = sq3.func("i", "sqlite3_config", "ii") +# int sqlite3_get_autocommit(sqlite3*); +sqlite3_get_autocommit = sq3.func("i", "sqlite3_get_autocommit", "p") +# int sqlite3_close_v2(sqlite3*); +sqlite3_close = sq3.func("i", "sqlite3_close_v2", "p") # int sqlite3_prepare( # sqlite3 *db, /* Database handle */ # const char *zSql, /* SQL statement, UTF-8 encoded */ @@ -14,7 +23,7 @@ # sqlite3_stmt **ppStmt, /* OUT: Statement handle */ # const char **pzTail /* OUT: Pointer to unused portion of zSql */ # ); -sqlite3_prepare = sq3.func("i", "sqlite3_prepare", "psipp") +sqlite3_prepare = sq3.func("i", "sqlite3_prepare_v2", "psipp") # int sqlite3_finalize(sqlite3_stmt *pStmt); sqlite3_finalize = sq3.func("i", "sqlite3_finalize", "p") # int sqlite3_step(sqlite3_stmt*); @@ -23,20 +32,17 @@ sqlite3_column_count = sq3.func("i", "sqlite3_column_count", "p") # int sqlite3_column_type(sqlite3_stmt*, int iCol); sqlite3_column_type = sq3.func("i", "sqlite3_column_type", "pi") +# int sqlite3_column_int(sqlite3_stmt*, int iCol); sqlite3_column_int = sq3.func("i", "sqlite3_column_int", "pi") -# using "d" return type gives wrong results +# double sqlite3_column_double(sqlite3_stmt*, int iCol); sqlite3_column_double = sq3.func("d", "sqlite3_column_double", "pi") +# const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); sqlite3_column_text = sq3.func("s", "sqlite3_column_text", "pi") # sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); -# TODO: should return long int -sqlite3_last_insert_rowid = sq3.func("i", "sqlite3_last_insert_rowid", "p") +sqlite3_last_insert_rowid = sq3.func("l", "sqlite3_last_insert_rowid", "p") # const char *sqlite3_errmsg(sqlite3*); sqlite3_errmsg = sq3.func("s", "sqlite3_errmsg", "p") -# Too recent -##const char *sqlite3_errstr(int); -# sqlite3_errstr = sq3.func("s", "sqlite3_errstr", "i") - SQLITE_OK = 0 SQLITE_ERROR = 1 @@ -51,6 +57,11 @@ SQLITE_BLOB = 4 SQLITE_NULL = 5 +SQLITE_CONFIG_URI = 17 + +# For compatibility with CPython sqlite3 driver +LEGACY_TRANSACTION_CONTROL = -1 + class Error(Exception): pass @@ -61,79 +72,142 @@ def check_error(db, s): raise Error(s, sqlite3_errmsg(db)) +def get_ptr_size(): + return uctypes.sizeof({"ptr": (0 | uctypes.PTR, uctypes.PTR)}) + + +def __prepare_stmt(db, sql): + # Prepares a statement + stmt_ptr = bytes(get_ptr_size()) + res = sqlite3_prepare(db, sql, -1, stmt_ptr, None) + check_error(db, res) + return int.from_bytes(stmt_ptr, sys.byteorder) + +def __exec_stmt(db, sql): + # Prepares, executes, and finalizes a statement + stmt = __prepare_stmt(db, sql) + sqlite3_step(stmt) + res = sqlite3_finalize(stmt) + check_error(db, res) + +def __is_dml(sql): + # Checks if a sql query is a DML, as these get a BEGIN in LEGACY_TRANSACTION_CONTROL + for dml in ["INSERT", "DELETE", "UPDATE", "MERGE"]: + if dml in sql.upper(): + return True + return False + + class Connections: - def __init__(self, h): - self.h = h + def __init__(self, db, isolation_level, autocommit): + self.db = db + self.isolation_level = isolation_level + self.autocommit = autocommit + + def commit(self): + if self.autocommit == LEGACY_TRANSACTION_CONTROL and not sqlite3_get_autocommit(self.db): + __exec_stmt(self.db, "COMMIT") + elif self.autocommit == False: + __exec_stmt(self.db, "COMMIT") + __exec_stmt(self.db, "BEGIN") + + def rollback(self): + if self.autocommit == LEGACY_TRANSACTION_CONTROL and not sqlite3_get_autocommit(self.db): + __exec_stmt(self.db, "ROLLBACK") + elif self.autocommit == False: + __exec_stmt(self.db, "ROLLBACK") + __exec_stmt(self.db, "BEGIN") def cursor(self): - return Cursor(self.h) + return Cursor(self.db, self.isolation_level, self.autocommit) def close(self): - s = sqlite3_close(self.h) - check_error(self.h, s) + if self.db: + if self.autocommit == False and not sqlite3_get_autocommit(self.db): + __exec_stmt(self.db, "ROLLBACK") + + res = sqlite3_close(self.db) + check_error(self.db, res) + self.db = None class Cursor: - def __init__(self, h): - self.h = h - self.stmnt = None + def __init__(self, db, isolation_level, autocommit): + self.db = db + self.isolation_level = isolation_level + self.autocommit = autocommit + self.stmt = None + + def __quote(val): + if isinstance(val, str): + return "'%s'" % val + return str(val) def execute(self, sql, params=None): + if self.stmt: + # If there is an existing statement, finalize that to free it + res = sqlite3_finalize(self.stmt) + check_error(self.db, res) + if params: - params = [quote(v) for v in params] + params = [self.__quote(v) for v in params] sql = sql % tuple(params) - print(sql) - b = bytearray(4) - s = sqlite3_prepare(self.h, sql, -1, b, None) - check_error(self.h, s) - self.stmnt = int.from_bytes(b, sys.byteorder) - # print("stmnt", self.stmnt) - self.num_cols = sqlite3_column_count(self.stmnt) - # print("num_cols", self.num_cols) - # If it's not select, actually execute it here - # num_cols == 0 for statements which don't return data (=> modify it) + + if __is_dml(sql) and self.autocommit == LEGACY_TRANSACTION_CONTROL and sqlite3_get_autocommit(self.db): + # For compatibility with CPython, add functionality for their default transaction + # behavior. Changing autocommit from LEGACY_TRANSACTION_CONTROL will remove this + __exec_stmt(self.db, "BEGIN " + self.isolation_level) + + self.stmt = __prepare_stmt(self.db, sql) + self.num_cols = sqlite3_column_count(self.stmt) + if not self.num_cols: v = self.fetchone() + # If it's not select, actually execute it here + # num_cols == 0 for statements which don't return data (=> modify it) assert v is None - self.lastrowid = sqlite3_last_insert_rowid(self.h) + self.lastrowid = sqlite3_last_insert_rowid(self.db) def close(self): - s = sqlite3_finalize(self.stmnt) - check_error(self.h, s) + if self.stmt: + res = sqlite3_finalize(self.stmt) + check_error(self.db, res) + self.stmt = None - def make_row(self): + def __make_row(self): res = [] for i in range(self.num_cols): - t = sqlite3_column_type(self.stmnt, i) - # print("type", t) + t = sqlite3_column_type(self.stmt, i) if t == SQLITE_INTEGER: - res.append(sqlite3_column_int(self.stmnt, i)) + res.append(sqlite3_column_int(self.stmt, i)) elif t == SQLITE_FLOAT: - res.append(sqlite3_column_double(self.stmnt, i)) + res.append(sqlite3_column_double(self.stmt, i)) elif t == SQLITE_TEXT: - res.append(sqlite3_column_text(self.stmnt, i)) + res.append(sqlite3_column_text(self.stmt, i)) else: raise NotImplementedError return tuple(res) def fetchone(self): - res = sqlite3_step(self.stmnt) - # print("step:", res) + res = sqlite3_step(self.stmt) if res == SQLITE_DONE: return None if res == SQLITE_ROW: - return self.make_row() - check_error(self.h, res) + return self.__make_row() + check_error(self.db, res) + + +def connect(fname, uri=False, isolation_level="", autocommit=LEGACY_TRANSACTION_CONTROL): + if isolation_level not in [None, "", "DEFERRED", "IMMEDIATE", "EXCLUSIVE"]: + raise Error("Invalid option for isolation level") + sqlite3_config(SQLITE_CONFIG_URI, int(uri)) -def connect(fname): - b = bytearray(4) - sqlite3_open(fname, b) - h = int.from_bytes(b, sys.byteorder) - return Connections(h) + sqlite_ptr = bytes(get_ptr_size()) + sqlite3_open(fname, sqlite_ptr) + db = int.from_bytes(sqlite_ptr, sys.byteorder) + if autocommit == False: + __exec_stmt(db, "BEGIN") -def quote(val): - if isinstance(val, str): - return "'%s'" % val - return str(val) + return Connections(db, isolation_level, autocommit) diff --git a/unix-ffi/sqlite3/test_sqlite3.py b/unix-ffi/sqlite3/test_sqlite3.py index 39dc07549..b168f18ff 100644 --- a/unix-ffi/sqlite3/test_sqlite3.py +++ b/unix-ffi/sqlite3/test_sqlite3.py @@ -17,3 +17,6 @@ assert row == e assert expected == [] + +cur.close() +conn.close() diff --git a/unix-ffi/sqlite3/test_sqlite3_2.py b/unix-ffi/sqlite3/test_sqlite3_2.py index 68a2abb86..515f865c3 100644 --- a/unix-ffi/sqlite3/test_sqlite3_2.py +++ b/unix-ffi/sqlite3/test_sqlite3_2.py @@ -10,3 +10,6 @@ cur.execute("SELECT * FROM foo") assert cur.fetchone() == (42,) assert cur.fetchone() is None + +cur.close() +conn.close() diff --git a/unix-ffi/sqlite3/test_sqlite3_3.py b/unix-ffi/sqlite3/test_sqlite3_3.py new file mode 100644 index 000000000..0a6fefc97 --- /dev/null +++ b/unix-ffi/sqlite3/test_sqlite3_3.py @@ -0,0 +1,42 @@ +import sqlite3 + + +def test_autocommit(): + conn = sqlite3.connect(":memory:", autocommit=True) + + # First cursor creates table and inserts value (DML) + cur = conn.cursor() + cur.execute("CREATE TABLE foo(a int)") + cur.execute("INSERT INTO foo VALUES (42)") + cur.close() + + # Second cursor fetches 42 due to the autocommit + cur = conn.cursor() + cur.execute("SELECT * FROM foo") + assert cur.fetchone() == (42,) + assert cur.fetchone() is None + + cur.close() + conn.close() + +def test_manual(): + conn = sqlite3.connect(":memory:", autocommit=False) + + # First cursor creates table, insert rolls back + cur = conn.cursor() + cur.execute("CREATE TABLE foo(a int)") + conn.commit() + cur.execute("INSERT INTO foo VALUES (42)") + cur.close() + conn.rollback() + + # Second connection fetches nothing due to the rollback + cur = conn.cursor() + cur.execute("SELECT * FROM foo") + assert cur.fetchone() is None + + cur.close() + conn.close() + +test_autocommit() +test_manual() diff --git a/unix-ffi/time/manifest.py b/unix-ffi/time/manifest.py index fcaaf7275..d1ff709a4 100644 --- a/unix-ffi/time/manifest.py +++ b/unix-ffi/time/manifest.py @@ -1,5 +1,5 @@ -metadata(version="0.5") +metadata(version="0.5.0") -require("ffilib", unix_ffi=True) +require("ffilib") module("time.py") diff --git a/unix-ffi/time/time.py b/unix-ffi/time/time.py index 075d904f5..319228dc8 100644 --- a/unix-ffi/time/time.py +++ b/unix-ffi/time/time.py @@ -1,6 +1,6 @@ from utime import * -from ucollections import namedtuple -import ustruct +from collections import namedtuple +import struct import uctypes import ffi import ffilib @@ -34,13 +34,13 @@ def _tuple_to_c_tm(t): - return ustruct.pack( + return struct.pack( "@iiiiiiiii", t[5], t[4], t[3], t[2], t[1] - 1, t[0] - 1900, (t[6] + 1) % 7, t[7] - 1, t[8] ) def _c_tm_to_tuple(tm): - t = ustruct.unpack("@iiiiiiiii", tm) + t = struct.unpack("@iiiiiiiii", tm) return _struct_time( t[5] + 1900, t[4] + 1, t[3], t[2], t[1], t[0], (t[6] - 1) % 7, t[7] + 1, t[8] ) @@ -64,7 +64,7 @@ def localtime(t=None): t = time() t = int(t) - a = ustruct.pack("l", t) + a = struct.pack("l", t) tm_p = localtime_(a) return _c_tm_to_tuple(uctypes.bytearray_at(tm_p, 36)) @@ -74,7 +74,7 @@ def gmtime(t=None): t = time() t = int(t) - a = ustruct.pack("l", t) + a = struct.pack("l", t) tm_p = gmtime_(a) return _c_tm_to_tuple(uctypes.bytearray_at(tm_p, 36)) diff --git a/unix-ffi/timeit/manifest.py b/unix-ffi/timeit/manifest.py index 82689bb8d..ea13af331 100644 --- a/unix-ffi/timeit/manifest.py +++ b/unix-ffi/timeit/manifest.py @@ -1,9 +1,9 @@ -metadata(version="3.3.3-3") +metadata(version="3.3.4") -require("getopt", unix_ffi=True) +require("getopt") require("itertools") # require("linecache") TODO -require("time", unix_ffi=True) +require("time") require("traceback") module("timeit.py") diff --git a/unix-ffi/ucurses/manifest.py b/unix-ffi/ucurses/manifest.py index 50648033e..8ec2675a5 100644 --- a/unix-ffi/ucurses/manifest.py +++ b/unix-ffi/ucurses/manifest.py @@ -2,8 +2,8 @@ # Originally written by Paul Sokolovsky. -require("os", unix_ffi=True) -require("tty", unix_ffi=True) -require("select", unix_ffi=True) +require("os") +require("tty") +require("select") package("ucurses") diff --git a/unix-ffi/urllib.parse/manifest.py b/unix-ffi/urllib.parse/manifest.py index 7023883f4..94109b134 100644 --- a/unix-ffi/urllib.parse/manifest.py +++ b/unix-ffi/urllib.parse/manifest.py @@ -1,6 +1,6 @@ metadata(version="0.5.2") -require("re", unix_ffi=True) +require("re") require("collections") require("collections-defaultdict")