diff --git a/datetime/datetime.py b/datetime/datetime.py index 03b4e4cdd..9ef43605e 100644 --- a/datetime/datetime.py +++ b/datetime/datetime.py @@ -914,6 +914,11 @@ class tzinfo: Subclasses must override the name(), utcoffset() and dst() methods. """ __slots__ = () + + def __new__(self, *args, **kwargs): + """Constructor.""" + return object.__new__(self) + def tzname(self, dt): "datetime -> string name of time zone." raise NotImplementedError("tzinfo subclass must override tzname()") diff --git a/fcntl/fcntl.py b/fcntl/fcntl.py index 5917840cb..9caad77c3 100644 --- a/fcntl/fcntl.py +++ b/fcntl/fcntl.py @@ -2,6 +2,75 @@ import os import ffilib +DN_ACCESS = 1 +DN_ATTRIB = 32 +DN_CREATE = 4 +DN_DELETE = 8 +DN_MODIFY = 2 +DN_MULTISHOT = 2147483648 +DN_RENAME = 16 +FASYNC = 8192 +FD_CLOEXEC = 1 +F_DUPFD = 0 +F_EXLCK = 4 +F_GETFD = 1 +F_GETFL = 3 +F_GETLEASE = 1025 +F_GETLK = 5 +F_GETLK64 = 5 +F_GETOWN = 9 +F_GETSIG = 11 +F_NOTIFY = 1026 +F_RDLCK = 0 +F_SETFD = 2 +F_SETFL = 4 +F_SETLEASE = 1024 +F_SETLK = 6 +F_SETLK64 = 6 +F_SETLKW = 7 +F_SETLKW64 = 7 +F_SETOWN = 8 +F_SETSIG = 10 +F_SHLCK = 8 +F_UNLCK = 2 +F_WRLCK = 1 +I_ATMARK = 21279 +I_CANPUT = 21282 +I_CKBAND = 21277 +I_FDINSERT = 21264 +I_FIND = 21259 +I_FLUSH = 21253 +I_FLUSHBAND = 21276 +I_GETBAND = 21278 +I_GETCLTIME = 21281 +I_GETSIG = 21258 +I_GRDOPT = 21255 +I_GWROPT = 21268 +I_LINK = 21260 +I_LIST = 21269 +I_LOOK = 21252 +I_NREAD = 21249 +I_PEEK = 21263 +I_PLINK = 21270 +I_POP = 21251 +I_PUNLINK = 21271 +I_PUSH = 21250 +I_RECVFD = 21262 +I_SENDFD = 21265 +I_SETCLTIME = 21280 +I_SETSIG = 21257 +I_SRDOPT = 21254 +I_STR = 21256 +I_SWROPT = 21267 +I_UNLINK = 21261 +LOCK_EX = 2 +LOCK_MAND = 32 +LOCK_NB = 4 +LOCK_READ = 64 +LOCK_RW = 192 +LOCK_SH = 1 +LOCK_UN = 8 +LOCK_WRITE = 128 libc = ffilib.libc() diff --git a/itertools/itertools.py b/itertools/itertools.py index 2ff053472..41ceb39b0 100644 --- a/itertools/itertools.py +++ b/itertools/itertools.py @@ -66,3 +66,12 @@ def accumulate(iterable, func=lambda x, y: x + y): for element in it: acc = func(acc, element) yield acc + +def product(*args, repeat=1): + if not args: + yield () + else: + args = args*repeat + for a in args[0]: + for prod in product(*args[1:]): + yield (a,)+prod diff --git a/logging/example_logging.py b/logging/example_logging.py index 0fefb8898..2ba2f1a18 100644 --- a/logging/example_logging.py +++ b/logging/example_logging.py @@ -4,3 +4,12 @@ log = logging.getLogger("test") log.debug("Test message: %d(%s)", 100, "foobar") log.info("Test message2: %d(%s)", 100, "foobar") +log.warning("Test message3: %d(%s)") +log.error("Test message4") +log.critical("Test message5") +logging.info("Test message6") + +try: + 1/0 +except: + log.exception("Some trouble (%s)", "expected") diff --git a/logging/logging.py b/logging/logging.py index 1c3ef0d84..f5b93da20 100644 --- a/logging/logging.py +++ b/logging/logging.py @@ -19,18 +19,30 @@ class Logger: + level = NOTSET + def __init__(self, name): - self.level = NOTSET self.name = name def _level_str(self, level): - if level in _level_dict: - return _level_dict[level] - return "LVL" + str(level) + l = _level_dict.get(level) + if l is not None: + return l + return "LVL%s" % level + + def setLevel(self, level): + self.level = level + + def isEnabledFor(self, level): + return level >= (self.level or _level) def log(self, level, msg, *args): if level >= (self.level or _level): - print(("%s:%s:" + msg) % ((self._level_str(level), self.name) + args), file=_stream) + _stream.write("%s:%s:" % (self._level_str(level), self.name)) + if not args: + print(msg, file=_stream) + else: + print(msg % args, file=_stream) def debug(self, msg, *args): self.log(DEBUG, msg, *args) @@ -47,6 +59,13 @@ def error(self, msg, *args): def critical(self, msg, *args): self.log(CRITICAL, msg, *args) + def exc(self, e, msg, *args): + self.log(ERROR, msg, *args) + sys.print_exception(e, _stream) + + def exception(self, msg, *args): + self.exc(sys.exc_info()[1], msg, *args) + _level = INFO _loggers = {} @@ -64,6 +83,9 @@ def info(msg, *args): def debug(msg, *args): getLogger(None).debug(msg, *args) +def error(msg, *args): + getLogger(None).error(msg, *args) + def basicConfig(level=INFO, filename=None, stream=None, format=None): global _level, _stream _level = level diff --git a/logging/metadata.txt b/logging/metadata.txt index c14869284..a1ff78f65 100644 --- a/logging/metadata.txt +++ b/logging/metadata.txt @@ -1,3 +1,3 @@ srctype = micropython-lib type = module -version = 0.1.3 +version = 0.3 diff --git a/logging/setup.py b/logging/setup.py index 20da4635b..ea7f3eb44 100644 --- a/logging/setup.py +++ b/logging/setup.py @@ -7,7 +7,7 @@ import sdist_upip setup(name='micropython-logging', - version='0.1.3', + version='0.3', description='logging module for MicroPython', long_description="This is a module reimplemented specifically for MicroPython standard library,\nwith efficient and lean design in mind. Note that this module is likely work\nin progress and likely supports just a subset of CPython's corresponding\nmodule. Please help with the development if you are interested in this\nmodule.", url='https://github.com/micropython/micropython-lib', diff --git a/multiprocessing/multiprocessing.py b/multiprocessing/multiprocessing.py index 470b50dbb..d18cf8b21 100644 --- a/multiprocessing/multiprocessing.py +++ b/multiprocessing/multiprocessing.py @@ -15,10 +15,12 @@ def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): def start(self): self.pid = os.fork() if not self.pid: - if self.r: - self.r.close() - self.target(*self.args, **self.kwargs) - os._exit(0) + try: + if self.r: + self.r.close() + self.target(*self.args, **self.kwargs) + finally: + os._exit(0) else: if self.w: self.w.close() @@ -37,7 +39,7 @@ class Connection: def __init__(self, fd): self.fd = fd - self.f = open(fd) + self.f = open(fd, 'b') def __repr__(self): return "" % self.f diff --git a/os.linux/metadata.txt b/os.linux/metadata.txt new file mode 100644 index 000000000..59295ca73 --- /dev/null +++ b/os.linux/metadata.txt @@ -0,0 +1,4 @@ +srctype = micropython-lib +type = package +version = 0.0.1 +author = Delio Brignoli diff --git a/os.linux/os/linux/__init__.py b/os.linux/os/linux/__init__.py new file mode 100644 index 000000000..9e2acbc80 --- /dev/null +++ b/os.linux/os/linux/__init__.py @@ -0,0 +1,60 @@ + +import ffilib +import os + + +libc = ffilib.libc() +sleep = libc.func('I', 'sleep', 'I') +_mount = libc.func('i', 'mount', 'sssLs') +_umount = libc.func('i', 'umount', 's') +_setenv = libc.func('i', 'setenv', 'ssi') +_reboot_syscall = libc.func('l', 'syscall', 'liiis') + + +LINUX_REBOOT_MAGIC1 = 0xfee1dead +LINUX_REBOOT_MAGIC2 = 672274793 +LINUX_REBOOT_CMD_RESTART = 0x01234567 +LINUX_REBOOT_CMD_HALT = 0xCDEF0123 +LINUX_REBOOT_CMD_CAD_ON = 0x89ABCDEF +LINUX_REBOOT_CMD_CAD_OFF = 0x00000000 +LINUX_REBOOT_CMD_POWER_OFF = 0x4321FEDC +LINUX_REBOOT_CMD_RESTART2 = 0xA1B2C3D4 +LINUX_REBOOT_CMD_SW_SUSPEND = 0xD000FCE2 +LINUX_REBOOT_CMD_KEXEC = 0x45584543 + + +def reboot(cmd, arg_str): + SYS_reboot = 142 + e = _reboot_syscall(SYS_reboot, LINUX_REBOOT_MAGIC1, LINUX_REBOOT_MAGIC2, cmd, arg_str) + os.check_error(e) + + +def mount(source, target, fstype, flags = 0, opts = None): + e = _mount(source, target, fstype, flags, opts) + os.check_error(e) + + +def umount(target): + e = _umount(target) + os.check_error(e) + + +def execv(path, args = []): + assert args, '`args` argument cannot be empty' + _args = [path] + args + [None] + _execl = libc.func('i', 'execl', 's'*len(_args)) + e = _execl(*_args) + os.check_error(e) + + +def execvp(executable, args = []): + assert args, '`args` argument cannot be empty' + _args = [executable] + args + [None] + _execlp = libc.func('i', 'execlp', 's'*len(_args)) + e = _execlp(*_args) + os.check_error(e) + + +def setenv(name, value, overwrite = True): + e = _setenv(name, value, 1 if overwrite else 0) + os.check_error(e) diff --git a/os.linux/os/linux/blkdev.py b/os.linux/os/linux/blkdev.py new file mode 100644 index 000000000..78f368299 --- /dev/null +++ b/os.linux/os/linux/blkdev.py @@ -0,0 +1,31 @@ + +# buffer data from src_fobj (it could be the stdin pipe) before writing +# it out in blocksz bytes chunks +def block_copy(src_fobj, dst_fobj, byte_count = 0, blocksz=512, progress_func=None): + buf = bytearray(blocksz) + buf_view = memoryview(buf) + read_bytes = 0 + left_to_read = byte_count + total_written_bytes = 0 + while True: + sz = src_fobj.readinto(buf_view[read_bytes:]) + #print('readinto() -> {}, {}'.format(read_bytes, sz), file=sys.stderr) + # in blocking mode sz will be zero only on EOF + left_to_read -= sz + read_bytes += sz + if not sz or (byte_count != 0 and left_to_read <= 0): + cnt = read_bytes + if byte_count != 0 and left_to_read <= 0: + cnt += left_to_read + dst_fobj.write(buf_view[:cnt]) + total_written_bytes += cnt + if progress_func is not None: + progress_func(total_written_bytes) + break + if read_bytes == blocksz: + dst_fobj.write(buf_view) + #print('write({})'.format(read_bytes), file=sys.stderr) + total_written_bytes += read_bytes + read_bytes = 0 + if progress_func is not None: + progress_func(total_written_bytes) diff --git a/os.linux/os/linux/evdev.py b/os.linux/os/linux/evdev.py new file mode 100644 index 000000000..ddedf6a8b --- /dev/null +++ b/os.linux/os/linux/evdev.py @@ -0,0 +1,18 @@ + +import builtins +import os +from os.linux import ioctl + +DEV_PATH_TPL = '/dev/input/event{}' + +KEY_SELECT = 0x161 + +def EVIOCGKEY(len): + return ioctl._ioc(ioctl._IOC_READ, ord('E'), 0x18, len) + +def get_global_keystate(dev, byte_array): + r = ioctl.ioctl_p(dev.fileno(), EVIOCGKEY(len(byte_array)), byte_array) + os.check_error(r) + +def open(index=0): + return builtins.open(DEV_PATH_TPL.format(index), 'r+b') diff --git a/os.linux/os/linux/i2c.py b/os.linux/os/linux/i2c.py new file mode 100644 index 000000000..28aeed439 --- /dev/null +++ b/os.linux/os/linux/i2c.py @@ -0,0 +1,13 @@ + +import builtins +from os.linux import ioctl + +DEV_PATH_TPL = '/dev/i2c-{}' + +I2C_SLAVE = 0x0703 + +def set_slave_addr(dev_fd, addr): + ioctl.ioctl_l(dev_fd, I2C_SLAVE, addr) + +def open(index=0): + return builtins.open(DEV_PATH_TPL.format(index), 'r+b') diff --git a/os.linux/os/linux/ioctl.py b/os.linux/os/linux/ioctl.py new file mode 100644 index 000000000..2af183573 --- /dev/null +++ b/os.linux/os/linux/ioctl.py @@ -0,0 +1,27 @@ + +import ffilib + +libc = ffilib.libc() + +TIOCCONS = 0x541D + +_IOC_WRITE = 1 +_IOC_READ = 2 + +_IOC_NRBITS = 8 +_IOC_TYPEBITS = 8 +_IOC_SIZEBITS = 14 +_IOC_DIRBITS = 2 + +_IOC_NRSHIFT = 0 +_IOC_TYPESHIFT = (_IOC_NRSHIFT+_IOC_NRBITS) +_IOC_SIZESHIFT = (_IOC_TYPESHIFT+_IOC_TYPEBITS) +_IOC_DIRSHIFT = (_IOC_SIZESHIFT+_IOC_SIZEBITS) + +def _ioc(dir, type, num, size): + return ((num << _IOC_NRSHIFT) | (type << _IOC_TYPESHIFT) | + (size << _IOC_SIZESHIFT) | (dir << _IOC_DIRSHIFT)) + +ioctl_p = libc.func("i", "ioctl", "iip") +ioctl_l = libc.func("i", "ioctl", "iil") +del libc diff --git a/os.linux/os/linux/mtd.py b/os.linux/os/linux/mtd.py new file mode 100644 index 000000000..6c12a2fb5 --- /dev/null +++ b/os.linux/os/linux/mtd.py @@ -0,0 +1,67 @@ + +import os +from os.linux import ioctl +try: + import ustruct as struct +except: + import struct + + +# struct erase_info_user { +# __u32 start; +# __u32 length; +# }; + +MTD_NORFLASH = 3 + +def _mtd_ioc(dir, num, size): + return ioctl._ioc(dir, ord('M'), num, size) + +MEM_INFO_STRUCT = 'BIIIIIII' +_MEM_INFO_LEN = struct.calcsize(MEM_INFO_STRUCT) +ERASE_INFO_STRUCT = 'II' +_ERASE_INFO_LEN = struct.calcsize(ERASE_INFO_STRUCT) +MEMGETINFO = _mtd_ioc(ioctl._IOC_READ, 1, _MEM_INFO_LEN) +MEMERASE = _mtd_ioc(ioctl._IOC_WRITE, 2, _ERASE_INFO_LEN) +MEMUNLOCK = _mtd_ioc(ioctl._IOC_WRITE, 6, _ERASE_INFO_LEN) + + +def _pack_erase_info(start, length): + return struct.pack(ERASE_INFO_STRUCT, start, length) + + +def _check_offset(offset, eblock): + assert offset % eblock == 0, \ + 'offset {} is not a multiple of erase block {}'.format(offset, eblock) + + +def info(mtd_dev): + mem_info = bytearray(_MEM_INFO_LEN) + e = ioctl.ioctl_p(mtd_dev.fileno(), MEMGETINFO, mem_info) + os.check_error(e) + # do not return padding words + return struct.unpack(MEM_INFO_STRUCT, mem_info)[:-2] + + +def unlock(mtd_dev, offset, eblock_size): + _check_offset(offset, eblock_size) + erase_info_user = _pack_erase_info(offset, eblock_size) + e = ioctl.ioctl_p(mtd_dev.fileno(), MEMUNLOCK, erase_info_user) + os.check_error(e) + + +def erase(mtd_dev, offset, eblock_size): + _check_offset(offset, eblock_size) + erase_info_user = _pack_erase_info(offset, eblock_size) + e = ioctl.ioctl_p(mtd_dev.fileno(), MEMERASE, erase_info_user) + os.check_error(e) + + +def read(mtd_dev, offset, length): + assert mtd_dev.seek(offset) == offset + return mtd_dev.read(length) + + +def write(mtd_dev, offset, data): + assert mtd_dev.seek(offset) == offset + return mtd_dev.write(data) diff --git a/os.linux/os/linux/spidev.py b/os.linux/os/linux/spidev.py new file mode 100644 index 000000000..ebf642ef3 --- /dev/null +++ b/os.linux/os/linux/spidev.py @@ -0,0 +1,17 @@ + +import builtins +from os.linux import ioctl + +DEV_PATH_TPL = '/dev/spidev{}.{}' + +def _spidev_ioc(dir, num, size): + return ioctl._ioc(dir, ord('k'), num, size) + +SPI_IOC_RD_BITS_PER_WORD = _spidev_ioc(ioctl._IOC_READ, 3, 1) +SPI_IOC_WR_BITS_PER_WORD = _spidev_ioc(ioctl._IOC_WRITE, 3, 1) + +SPI_IOC_RD_MAX_SPEED_HZ = _spidev_ioc(ioctl._IOC_READ, 4, 4) +SPI_IOC_WR_MAX_SPEED_HZ = _spidev_ioc(ioctl._IOC_WRITE, 4, 4) + +def open(bus_idx=0, cs_idx=0): + return builtins.open(DEV_PATH_TPL.format(bus_idx, cs_idx), 'r+b') diff --git a/os.linux/os/linux/syslog.py b/os.linux/os/linux/syslog.py new file mode 100644 index 000000000..81d1e27ba --- /dev/null +++ b/os.linux/os/linux/syslog.py @@ -0,0 +1,99 @@ + +import sys +import os +from os import libc + +openlog_ = libc.func("v", "openlog", "sii") +setlogmask_ = libc.func("i", "setlogmask", "i") +syslog_ = libc.func("v", "syslog", "is") +isatty_ = libc.func("i", "isatty", "i") + +# Syslog priorities +CRITICAL = 2 +ERROR = 3 +WARNING = 4 +NOTICE = 5 +INFO = 6 +DEBUG = 7 +NOTSET = 0 + +# Facility codes +LOG_USER = (1<<3) # random user-level messages + +# Option flags for openlog +LOG_PID = 0x01 # log the pid with each message +LOG_CONS = 0x02 # log on the console if errors in sending +LOG_ODELAY = 0x04 # delay open until first syslog() (default) +LOG_NDELAY = 0x08 # don't delay open +LOG_NOWAIT = 0x10 # don't wait for console forks: DEPRECATED +LOG_PERROR = 0x20 # log to stderr as well + +def _logmask_upto(pri): + return ((1<<((pri)+1))-1) + +class Logger: + + def __init__(self, name): + self.name = name + + def log(self, level, msg, *args): + if self.name is not None: + s = ('%s:'+ msg) % ((self.name,) + args) + else: + s = msg % args + syslog_(level, s) + + def debug(self, msg, *args): + self.log(DEBUG, msg, *args) + + def info(self, msg, *args): + self.log(INFO, msg, *args) + + def warning(self, msg, *args): + self.log(WARNING, msg, *args) + + def error(self, msg, *args): + self.log(ERROR, msg, *args) + + def critical(self, msg, *args): + self.log(CRITICAL, msg, *args) + + +_level = ERROR +_loggers = {} + +r = isatty_(sys.stdout.fileno()) +os.check_error(r) + +flags = LOG_CONS | LOG_PID +# if we are outputting to a tty log also to stderr +if r > 0: + flags |= LOG_PERROR +ident = 'python' +if len(sys.argv): + ident = sys.argv[0] +openlog_(ident, flags, LOG_USER) + +r = setlogmask_(_logmask_upto(_level)) +os.check_error(r) + +def getLogger(name): + if name in _loggers: + return _loggers[name] + l = Logger(name) + _loggers[name] = l + return l + +def info(msg, *args): + getLogger(None).info(msg, *args) + +def debug(msg, *args): + getLogger(None).debug(msg, *args) + +def basicConfig(level=INFO, filename=None, format=None): + global _level + _level = level + if filename is not None: + print("logging.basicConfig: filename arg is not supported") + if format is not None: + print("logging.basicConfig: format arg is not supported") diff --git a/os.linux/os/linux/time.py b/os.linux/os/linux/time.py new file mode 100644 index 000000000..694bf2322 --- /dev/null +++ b/os.linux/os/linux/time.py @@ -0,0 +1,40 @@ + +import ustruct +import ffilib +import os + + +TIMESPEC_FMT = 'll' +TIMESPEC_SIZE = ustruct.calcsize(TIMESPEC_FMT) +TIMEVAL_FMT = TIMESPEC_FMT +TIMEVAL_SIZE = ustruct.calcsize(TIMEVAL_FMT) + +CLOCK_MONOTONIC = 1 +CLOCK_MONOTONIC_RAW = 4 + +librt = ffilib.open('librt') +_clock_gettime = librt.func('i', 'clock_gettime', 'ip') +_gettimeofday = librt.func('i', 'gettimeofday', 'ip') +_ts_buf = bytearray(TIMESPEC_SIZE) +_tv_buf = bytearray(TIMEVAL_SIZE) + +def clock_gettimeofday(clk_id): + e1 = _clock_gettime(clk_id, _ts_buf) + e2 = _gettimeofday(_tv_buf, None) + os.check_error(e1) + os.check_error(e2) + s, ns = ustruct.unpack(TIMESPEC_FMT, _ts_buf) + utc_s, utc_us = ustruct.unpack(TIMEVAL_FMT, _tv_buf) + return s, ns, utc_s, utc_us + +def clock_gettime(clk_id): + e = _clock_gettime(clk_id, _ts_buf) + os.check_error(e) + s, ns = ustruct.unpack(TIMESPEC_FMT, _ts_buf) + return (s*10**9)+ns + +def monotime(): + return clock_gettime(CLOCK_MONOTONIC) + +def monotime_raw(): + return clock_gettime(CLOCK_MONOTONIC_RAW) diff --git a/os.linux/setup.py b/os.linux/setup.py new file mode 100644 index 000000000..aa8214738 --- /dev/null +++ b/os.linux/setup.py @@ -0,0 +1,17 @@ +import sys +# Remove current dir from sys.path, otherwise setuptools will peek up our +# module instead of system. +sys.path.pop(0) +from setuptools import setup + + +setup(name='micropython-os.linux', + version='0.0.1', + description='os.linux module for MicroPython', + long_description="Linux specific OS functions not included in micropython-lib's os module", + author='Delio Brignoli', + author_email='dbrignoli@audioscience.com', + maintainer='Delio Brignoli', + maintainer_email='dbrignoli@audioscience.com', + license='MIT', + packages=['os.linux']) diff --git a/os/os/__init__.py b/os/os/__init__.py index f941e7f54..b88ee9f9c 100644 --- a/os/os/__init__.py +++ b/os/os/__init__.py @@ -21,6 +21,11 @@ O_APPEND = 0o0002000 O_NONBLOCK = 0o0004000 +WNOHANG = 0x00000001 + +SIGKILL = 9 +SIGTERM = 15 + error = OSError name = "posix" sep = "/" @@ -55,6 +60,7 @@ execvp_ = libc.func("i", "execvp", "PP") kill_ = libc.func("i", "kill", "ii") getenv_ = libc.func("s", "getenv", "P") + setpgid_ = libc.func("i", "setpgid", "ii") @@ -148,7 +154,7 @@ def walk(top, topdown=True): files = [] dirs = [] for dirent in ilistdir(top): - mode = dirent[1] << 12 + mode = dirent[1] fname = fsdecode(dirent[0]) if stat_.S_ISDIR(mode): if fname != "." and fname != "..": @@ -232,6 +238,11 @@ def kill(pid, sig): r = kill_(pid, sig) check_error(r) +def setpgid(pid, pgid): + r = setpgid_(pid, pgid) + check_error(r) + return r + def system(command): r = system_(command) check_error(r) diff --git a/uasyncio.core/metadata.txt b/uasyncio.core/metadata.txt index 6d40c7a33..21d581668 100644 --- a/uasyncio.core/metadata.txt +++ b/uasyncio.core/metadata.txt @@ -1,6 +1,6 @@ srctype = micropython-lib type = package -version = 1.7.2 +version = 2.0 author = Paul Sokolovsky desc = Lightweight asyncio-like library for MicroPython, built around native Python coroutines. (Core event loop). long_desc = Lightweight asyncio-like library for MicroPython, built around native Python coroutines. (Core event loop). diff --git a/uasyncio.core/setup.py b/uasyncio.core/setup.py index 3235a6dcc..d7c468099 100644 --- a/uasyncio.core/setup.py +++ b/uasyncio.core/setup.py @@ -7,7 +7,7 @@ import sdist_upip setup(name='micropython-uasyncio.core', - version='1.7.2', + version='2.0', description='Lightweight asyncio-like library for MicroPython, built around native Python coroutines. (Core event loop).', long_description='Lightweight asyncio-like library for MicroPython, built around native Python coroutines. (Core event loop).', url='https://github.com/micropython/micropython-lib', diff --git a/uasyncio.core/test_full_wait.py b/uasyncio.core/test_full_wait.py index 8954a9151..17af6f26d 100644 --- a/uasyncio.core/test_full_wait.py +++ b/uasyncio.core/test_full_wait.py @@ -47,4 +47,4 @@ def cb_2nd(): print(loop.msgs) # .wait() is now called on each loop iteration, and for our mock case, it means that # at the time of running, self.time() will be skewed by 100 virtual time units. -assert loop.msgs == ['I should be run first, time: 200', 'I should be run second, time: 600'], str(loop.msgs) +assert loop.msgs == ['I should be run first, time: 100', 'I should be run second, time: 500'], str(loop.msgs) diff --git a/uasyncio.core/uasyncio/core.py b/uasyncio.core/uasyncio/core.py index 274883a7f..e8fa57252 100644 --- a/uasyncio.core/uasyncio/core.py +++ b/uasyncio.core/uasyncio/core.py @@ -1,5 +1,6 @@ import utime as time import utimeq +import ucollections type_gen = type((lambda: (yield))()) @@ -13,6 +14,7 @@ def set_debug(val): if val: import logging log = logging.getLogger("uasyncio.core") + log.setLevel(val) class CancelledError(Exception): @@ -25,8 +27,9 @@ class TimeoutError(CancelledError): class EventLoop: - def __init__(self, len=42): - self.q = utimeq.utimeq(len) + def __init__(self, runq_len=16, waitq_len=16): + self.runq = ucollections.deque((), runq_len, True) + self.waitq = utimeq.utimeq(waitq_len) # Current task being run. Task is a top-level coroutine scheduled # in the event loop (sub-coroutines executed transparently by # yield from/await, event loop "doesn't see" them). @@ -41,18 +44,24 @@ def create_task(self, coro): # CPython asyncio incompatibility: we don't return Task object def call_soon(self, callback, *args): - self.call_at_(self.time(), callback, args) + if __debug__ and DEBUG: + log.debug("Scheduling in runq: %s", (callback, args)) + self.runq.append(callback) + if not isinstance(callback, type_gen): + self.runq.append(args) def call_later(self, delay, callback, *args): self.call_at_(time.ticks_add(self.time(), int(delay * 1000)), callback, args) def call_later_ms(self, delay, callback, *args): + if not delay: + return self.call_soon(callback, *args) self.call_at_(time.ticks_add(self.time(), delay), callback, args) def call_at_(self, time, callback, args=()): if __debug__ and DEBUG: - log.debug("Scheduling %s", (time, callback, args)) - self.q.push(time, callback, args) + log.debug("Scheduling in waitq: %s", (time, callback, args)) + self.waitq.push(time, callback, args) def wait(self, delay): # Default wait implementation, to be overriden in subclasses @@ -61,64 +70,85 @@ def wait(self, delay): log.debug("Sleeping for: %s", delay) time.sleep_ms(delay) + def cancel(self, coro, exc = CancelledError): + if isinstance(coro, type_gen): + try: + prev = coro.pend_throw(exc) + if __debug__ and DEBUG: + log.debug("Cancelling %s asynchronously", coro) + if prev is None: + _event_loop.remove_polled_cb(coro) + _event_loop.call_soon(coro) + except TypeError: + # .pend_throw() works only on started coroutines + # Kill the coro right here, right now + # No need to worry about IO because the coro cannot be registered + # because it didn't get a chance to run yet + try: + if __debug__ and DEBUG: + log.debug("Cancelling %s synchronously", coro) + coro.throw(exc) + except exc: + pass + else: + raise UnimplementedError('Cancelling a callback is not supported') + def run_forever(self): cur_task = [0, 0, 0] while True: - if self.q: - # wait() may finish prematurely due to I/O completion, - # and schedule new, earlier than before tasks to run. - while 1: - t = self.q.peektime() - tnow = self.time() - delay = time.ticks_diff(t, tnow) - if delay < 0: - delay = 0 - # Always call wait(), to give a chance to I/O scheduling - self.wait(delay) - if delay == 0: - break - - self.q.pop(cur_task) - t = cur_task[0] - cb = cur_task[1] - args = cur_task[2] + # Expire entries in waitq and move them to runq + tnow = self.time() + while self.waitq: + t = self.waitq.peektime() + delay = time.ticks_diff(t, tnow) + if delay > 0: + break + self.waitq.pop(cur_task) if __debug__ and DEBUG: - log.debug("Next coroutine to run: %s", (t, cb, args)) + log.debug("Moving from waitq to runq: %s", cur_task[1]) + self.call_soon(cur_task[1], *cur_task[2]) + + # Process runq + l = len(self.runq) + if __debug__ and DEBUG: + log.debug("Entries in runq: %d", l) + while l: + cb = self.runq.popleft() + l -= 1 + args = () + if not isinstance(cb, type_gen): + args = self.runq.popleft() + l -= 1 + if __debug__ and DEBUG: + log.info("Next callback to run: %s", (cb, args)) + cb(*args) + continue + + if __debug__ and DEBUG: + log.info("Next coroutine to run: %s", (cb, args)) self.cur_task = cb -# __main__.mem_info() - else: - self.wait(-1) - # Assuming IO completion scheduled some tasks - continue - if callable(cb): - cb(*args) - else: delay = 0 try: - if __debug__ and DEBUG: - log.debug("Coroutine %s send args: %s", cb, args) - if args == (): + if args is (): ret = next(cb) else: ret = cb.send(*args) if __debug__ and DEBUG: - log.debug("Coroutine %s yield result: %s", cb, ret) + log.info("Coroutine %s yield result: %s", cb, ret) if isinstance(ret, SysCall1): arg = ret.arg if isinstance(ret, SleepMs): delay = arg elif isinstance(ret, IORead): - cb.pend_throw(False) - self.add_reader(arg, cb) + self.add_reader(arg.fileno(), cb) continue elif isinstance(ret, IOWrite): - cb.pend_throw(False) - self.add_writer(arg, cb) + self.add_writer(arg.fileno(), cb) continue elif isinstance(ret, IOReadDone): - self.remove_reader(arg) + self.remove_reader(arg.fileno(), cb) elif isinstance(ret, IOWriteDone): - self.remove_writer(arg) + self.remove_writer(arg.fileno(), cb) elif isinstance(ret, StopLoop): return arg else: @@ -139,15 +169,38 @@ def run_forever(self): except StopIteration as e: if __debug__ and DEBUG: log.debug("Coroutine finished: %s", cb) + self.remove_polled_cb(cb) continue except CancelledError as e: if __debug__ and DEBUG: log.debug("Coroutine cancelled: %s", cb) + self.remove_polled_cb(cb) continue + except Exception as e: + if log: + log.error("Coroutine %s exception: %s", cb, e) # Currently all syscalls don't return anything, so we don't # need to feed anything to the next invocation of coroutine. # If that changes, need to pass that value below. - self.call_later_ms(delay, cb) + if delay: + self.call_later_ms(delay, cb) + else: + self.call_soon(cb) + + # Wait until next waitq task or I/O availability + delay = 0 + if not self.runq: + if self.waitq: + tnow = self.time() + t = self.waitq.peektime() + delay = time.ticks_diff(t, tnow) + if delay < 0: + delay = 0 + else: + if __debug__ and DEBUG: + log.info("No more tasks to execute, waiting forever") + delay = -1 + self.wait(delay) def run_until_complete(self, coro): def _run_and_stop(): @@ -195,10 +248,10 @@ class IOWriteDone(SysCall1): _event_loop = None _event_loop_class = EventLoop -def get_event_loop(len=42): +def get_event_loop(runq_len=16, waitq_len=16): global _event_loop if _event_loop is None: - _event_loop = _event_loop_class(len) + _event_loop = _event_loop_class(runq_len, waitq_len) return _event_loop def sleep(secs): @@ -234,10 +287,8 @@ def __next__(self): sleep_ms = SleepMs() -def cancel(coro): - prev = coro.pend_throw(CancelledError()) - if prev is False: - _event_loop.call_soon(coro) +def cancel(coro, exc=CancelledError): + _event_loop.cancel(coro, exc=exc) class TimeoutObj: @@ -258,10 +309,7 @@ def timeout_func(timeout_obj): if timeout_obj.coro: if __debug__ and DEBUG: log.debug("timeout_func: cancelling %s", timeout_obj.coro) - prev = timeout_obj.coro.pend_throw(TimeoutError()) - #print("prev pend", prev) - if prev is False: - _event_loop.call_soon(timeout_obj.coro) + cancel(timeout_obj.coro, exc=TimeoutError()) timeout_obj = TimeoutObj(_event_loop.cur_task) _event_loop.call_later_ms(timeout, timeout_func, timeout_obj) diff --git a/uasyncio.udp/metadata.txt b/uasyncio.udp/metadata.txt index 0230fb707..c791cef1b 100644 --- a/uasyncio.udp/metadata.txt +++ b/uasyncio.udp/metadata.txt @@ -1,6 +1,6 @@ srctype = micropython-lib type = package -version = 0.1 +version = 0.1.1 author = Paul Sokolovsky desc = UDP support for MicroPython's uasyncio depends = uasyncio diff --git a/uasyncio.udp/setup.py b/uasyncio.udp/setup.py index 3d8335151..95ce6a027 100644 --- a/uasyncio.udp/setup.py +++ b/uasyncio.udp/setup.py @@ -7,7 +7,7 @@ import sdist_upip setup(name='micropython-uasyncio.udp', - version='0.1', + version='0.1.1', description="UDP support for MicroPython's uasyncio", long_description="This is a module reimplemented specifically for MicroPython standard library,\nwith efficient and lean design in mind. Note that this module is likely work\nin progress and likely supports just a subset of CPython's corresponding\nmodule. Please help with the development if you are interested in this\nmodule.", url='https://github.com/micropython/micropython-lib', diff --git a/uasyncio.udp/uasyncio/udp.py b/uasyncio.udp/uasyncio/udp.py index dfa4f879d..5987bf7d2 100644 --- a/uasyncio.udp/uasyncio/udp.py +++ b/uasyncio.udp/uasyncio/udp.py @@ -45,7 +45,7 @@ def recvfrom(s, n): def sendto(s, buf, addr=None): while 1: - res = s.sendto(buf, 0, addr) + res = s.sendto(buf, addr) #print("send res:", res) if res == len(buf): return diff --git a/uasyncio/benchmark/boom_uasyncio.py b/uasyncio/benchmark/boom_uasyncio.py index def5bfb19..9f2654aaf 100644 --- a/uasyncio/benchmark/boom_uasyncio.py +++ b/uasyncio/benchmark/boom_uasyncio.py @@ -19,7 +19,7 @@ def validate(resp): no = int(l.split()[1]) seen.append(no) c = t.count(l + "\r\n") - assert c == 400101 + assert c == 400101, str(c) assert t.endswith("=== END ===") cnt += 1 diff --git a/uasyncio/metadata.txt b/uasyncio/metadata.txt index 68ceb4b92..c0cbd68bf 100644 --- a/uasyncio/metadata.txt +++ b/uasyncio/metadata.txt @@ -1,6 +1,6 @@ srctype = micropython-lib type = package -version = 1.4.2 +version = 2.0 author = Paul Sokolovsky desc = Lightweight asyncio-like library for MicroPython, built around native Python coroutines. long_desc = README.rst diff --git a/uasyncio/setup.py b/uasyncio/setup.py index 6dc73ce43..8bb6fa91b 100644 --- a/uasyncio/setup.py +++ b/uasyncio/setup.py @@ -7,7 +7,7 @@ import sdist_upip setup(name='micropython-uasyncio', - version='1.4.2', + version='2.0', description='Lightweight asyncio-like library for MicroPython, built around native Python coroutines.', long_description=open('README.rst').read(), url='https://github.com/micropython/micropython-lib', diff --git a/uasyncio/uasyncio/__init__.py b/uasyncio/uasyncio/__init__.py index e26757a25..0496e3129 100644 --- a/uasyncio/uasyncio/__init__.py +++ b/uasyncio/uasyncio/__init__.py @@ -1,4 +1,4 @@ -import uerrno +import uerrno as errno import uselect as select import usocket as _socket from uasyncio.core import * @@ -13,140 +13,145 @@ def set_debug(val): if val: import logging log = logging.getLogger("uasyncio") + log.setLevel(val) -class PollEventLoop(EventLoop): +class EpollEventLoop(EventLoop): - def __init__(self, len=42): - EventLoop.__init__(self, len) + def __init__(self, runq_len=16, waitq_len=16): + EventLoop.__init__(self, runq_len, waitq_len) self.poller = select.poll() self.objmap = {} - def add_reader(self, sock, cb, *args): - if DEBUG and __debug__: - log.debug("add_reader%s", (sock, cb, args)) + def _unregister_fd(self, fd): + try: + self.poller.unregister(fd) + except OSError as e: + if e.args[0] == errno.ENOENT: + if __debug__ and DEBUG: + log.debug("_unregister_fd() attepted to remove unregistered FD %s", fd) + else: + raise + + def remove_polled_cb(self, cb): + _id = id(cb) + for fd, cbs in self.objmap.items(): + cbs.pop(id(cb), None) + if not cbs: + self._unregister_fd(fd) + + def add_reader(self, fd, cb, *args): + if __debug__ and DEBUG: + log.debug("add_reader%s", (fd, cb, args)) + cbs = self.objmap.setdefault(fd, {}) + self.poller.register(fd, select.POLLIN) if args: - self.poller.register(sock, select.POLLIN) - self.objmap[id(sock)] = (cb, args) + cbs[id(cb)] = (cb, args) else: - self.poller.register(sock, select.POLLIN) - self.objmap[id(sock)] = cb - - def remove_reader(self, sock): - if DEBUG and __debug__: - log.debug("remove_reader(%s)", sock) - self.poller.unregister(sock) - del self.objmap[id(sock)] - - def add_writer(self, sock, cb, *args): - if DEBUG and __debug__: - log.debug("add_writer%s", (sock, cb, args)) + cbs[id(cb)] = (cb, None) + + def remove_reader(self, fd, cb): + if __debug__ and DEBUG: + log.debug("remove_reader(%s)", (fd, cb)) + cbs = self.objmap.get(fd, {}) + cbs.pop(id(cb), None) + if not cbs: + self._unregister_fd(fd) + + def add_writer(self, fd, cb, *args): + if __debug__ and DEBUG: + log.debug("add_writer%s", (fd, cb, args)) + cbs = self.objmap.setdefault(fd, {}) + self.poller.register(fd, select.POLLOUT) if args: - self.poller.register(sock, select.POLLOUT) - self.objmap[id(sock)] = (cb, args) + cbs[id(cb)] = (cb, args) else: - self.poller.register(sock, select.POLLOUT) - self.objmap[id(sock)] = cb + cbs[id(cb)] = (cb, None) - def remove_writer(self, sock): - if DEBUG and __debug__: - log.debug("remove_writer(%s)", sock) - try: - self.poller.unregister(sock) - self.objmap.pop(id(sock), None) - except OSError as e: - # StreamWriter.awrite() first tries to write to a socket, - # and if that succeeds, yield IOWrite may never be called - # for that socket, and it will never be added to poller. So, - # ignore such error. - if e.args[0] != uerrno.ENOENT: - raise + def remove_writer(self, fd, cb): + if __debug__ and DEBUG: + log.debug("remove_writer(%s)", fd) + cbs = self.objmap.get(fd, {}) + cbs.pop(id(cb), None) + if not cbs: + self._unregister_fd(fd) def wait(self, delay): - if DEBUG and __debug__: - log.debug("poll.wait(%d)", delay) + if __debug__ and DEBUG: + log.debug("epoll.wait(%s)", delay) + for fd, cbs in self.objmap.items(): + for cb, args in cbs.values(): + log.debug("epoll.registered(%d) %s", fd, (cb, args)) + # We need one-shot behavior (second arg of 1 to .poll()) - res = self.poller.ipoll(delay, 1) - #log.debug("poll result: %s", res) - # Remove "if res" workaround after - # https://github.com/micropython/micropython/issues/2716 fixed. - if res: - for sock, ev in res: - cb = self.objmap[id(sock)] - if ev & (select.POLLHUP | select.POLLERR): - # These events are returned even if not requested, and - # are sticky, i.e. will be returned again and again. - # If the caller doesn't do proper error handling and - # unregister this sock, we'll busy-loop on it, so we - # as well can unregister it now "just in case". - self.remove_reader(sock) - if DEBUG and __debug__: - log.debug("Calling IO callback: %r", cb) - if isinstance(cb, tuple): - cb[0](*cb[1]) - else: - cb.pend_throw(None) + res = self.poller.poll(delay, 1) + #log.debug("epoll result: %s", res) + for fd, ev in res: + # Remove the registered callbacks dictionary from its parent + # so when callbacks are invoked they can add their registrations + # to a fresh dictionary. + cbs = self.objmap.pop(fd, {}) + if not cbs: + log.error("Event %d on fd %r but no callback registered", ev, fd) + continue + if __debug__ and DEBUG: + s = '\n'.join(str(v) for v in cbs.values()) + log.debug("Matching IO callbacks for %r:\n%s", (fd, ev), s) + while cbs: + _id, data = cbs.popitem() + cb, args = data + if args is None: + if __debug__ and DEBUG: + log.debug("Scheduling IO coro: %r", (fd, ev, cb)) self.call_soon(cb) + else: + if __debug__ and DEBUG: + log.debug("Calling IO callback: %r", (fd, ev, cb, args)) + cb(*args) + # If no callback registered an event for this fd unregister it + if not self.objmap.get(fd, None): + self._unregister_fd(fd) class StreamReader: - def __init__(self, polls, ios=None): - if ios is None: - ios = polls - self.polls = polls - self.ios = ios + def __init__(self, s): + self.s = s def read(self, n=-1): + yield IORead(self.s) while True: - yield IORead(self.polls) - res = self.ios.read(n) + res = self.s.read(n) if res is not None: break - # This should not happen for real sockets, but can easily - # happen for stream wrappers (ssl, websockets, etc.) - #log.warn("Empty read") + log.warning("Empty read") if not res: - yield IOReadDone(self.polls) + yield IOReadDone(self.s) return res - def readexactly(self, n): - buf = b"" - while n: - yield IORead(self.polls) - res = self.ios.read(n) - assert res is not None - if not res: - yield IOReadDone(self.polls) - break - buf += res - n -= len(res) - return buf - def readline(self): - if DEBUG and __debug__: + if __debug__ and DEBUG: log.debug("StreamReader.readline()") - buf = b"" + yield IORead(self.s) +# if __debug__ and DEBUG: +# log.debug("StreamReader.readline(): after IORead: %s", s) while True: - yield IORead(self.polls) - res = self.ios.readline() - assert res is not None - if not res: - yield IOReadDone(self.polls) - break - buf += res - if buf[-1] == 0x0a: + res = self.s.readline() + if res is not None: break - if DEBUG and __debug__: - log.debug("StreamReader.readline(): %s", buf) - return buf + log.warning("Empty read") + if not res: + yield IOReadDone(self.s) + if __debug__ and DEBUG: + log.debug("StreamReader.readline(): res: %s", res) + return res def aclose(self): - yield IOReadDone(self.polls) - self.ios.close() + yield IOReadDone(self.s) + self.s.close() def __repr__(self): - return "" % (self.polls, self.ios) + return "" % self.s class StreamWriter: @@ -155,40 +160,34 @@ def __init__(self, s, extra): self.s = s self.extra = extra - def awrite(self, buf, off=0, sz=-1): + def awrite(self, buf): # This method is called awrite (async write) to not proliferate # incompatibility with original asyncio. Unlike original asyncio # whose .write() method is both not a coroutine and guaranteed # to return immediately (which means it has to buffer all the # data), this method is a coroutine. - if sz == -1: - sz = len(buf) - off - if DEBUG and __debug__: + sz = len(buf) + if __debug__ and DEBUG: log.debug("StreamWriter.awrite(): spooling %d bytes", sz) while True: - res = self.s.write(buf, off, sz) + res = self.s.write(buf) # If we spooled everything, return immediately if res == sz: - if DEBUG and __debug__: + if __debug__ and DEBUG: log.debug("StreamWriter.awrite(): completed spooling %d bytes", res) return if res is None: res = 0 - if DEBUG and __debug__: + if __debug__ and DEBUG: log.debug("StreamWriter.awrite(): spooled partial %d bytes", res) assert res < sz - off += res + buf = buf[res:] sz -= res yield IOWrite(self.s) #assert s2.fileno() == self.s.fileno() - if DEBUG and __debug__: + if __debug__ and DEBUG: log.debug("StreamWriter.awrite(): can write more") - # Write piecewise content from iterable (usually, a generator) - def awriteiter(self, iterable): - for buf in iterable: - yield from self.awrite(buf) - def aclose(self): yield IOWriteDone(self.s) self.s.close() @@ -200,32 +199,25 @@ def __repr__(self): return "" % self.s -def open_connection(host, port, ssl=False): - if DEBUG and __debug__: +def open_connection(host, port): + if __debug__ and DEBUG: log.debug("open_connection(%s, %s)", host, port) - ai = _socket.getaddrinfo(host, port, 0, _socket.SOCK_STREAM) - ai = ai[0] - s = _socket.socket(ai[0], ai[1], ai[2]) + s = _socket.socket() s.setblocking(False) + ai = _socket.getaddrinfo(host, port) + addr = ai[0][4] try: - s.connect(ai[-1]) + s.connect(addr) except OSError as e: - if e.args[0] != uerrno.EINPROGRESS: + if e.args[0] != errno.EINPROGRESS: raise - if DEBUG and __debug__: + if __debug__ and DEBUG: log.debug("open_connection: After connect") yield IOWrite(s) -# if __debug__: +# if __debug__ and DEBUG: # assert s2.fileno() == s.fileno() - if DEBUG and __debug__: + if __debug__ and DEBUG: log.debug("open_connection: After iowait: %s", s) - if ssl: - print("Warning: uasyncio SSL support is alpha") - import ussl - s.setblocking(True) - s2 = ussl.wrap_socket(s) - s.setblocking(False) - return StreamReader(s, s2), StreamWriter(s2, {}) return StreamReader(s), StreamWriter(s, {}) @@ -235,24 +227,26 @@ def start_server(client_coro, host, port, backlog=10): ai = _socket.getaddrinfo(host, port, 0, _socket.SOCK_STREAM) ai = ai[0] s = _socket.socket(ai[0], ai[1], ai[2]) - s.setblocking(False) - - s.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) - s.bind(ai[-1]) - s.listen(backlog) - while True: - if DEBUG and __debug__: - log.debug("start_server: Before accept") - yield IORead(s) - if DEBUG and __debug__: - log.debug("start_server: After iowait") - s2, client_addr = s.accept() - s2.setblocking(False) - if DEBUG and __debug__: - log.debug("start_server: After accept: %s", s2) - extra = {"peername": client_addr} - yield client_coro(StreamReader(s2), StreamWriter(s2, extra)) + try: + s.setblocking(False) + s.setsockopt(_socket.SOL_SOCKET, _socket.SO_REUSEADDR, 1) + s.bind(ai[-1]) + s.listen(backlog) + while True: + if DEBUG and __debug__: + log.debug("start_server: Before accept") + yield IORead(s) + if DEBUG and __debug__: + log.debug("start_server: After iowait") + s2, client_addr = s.accept() + s2.setblocking(False) + if DEBUG and __debug__: + log.debug("start_server: After accept: %s", s2) + extra = {"peername": client_addr} + yield client_coro(StreamReader(s2), StreamWriter(s2, extra)) + finally: + s.close() import uasyncio.core -uasyncio.core._event_loop_class = PollEventLoop +uasyncio.core._event_loop_class = EpollEventLoop diff --git a/ubus/metadata.txt b/ubus/metadata.txt new file mode 100644 index 000000000..59295ca73 --- /dev/null +++ b/ubus/metadata.txt @@ -0,0 +1,4 @@ +srctype = micropython-lib +type = package +version = 0.0.1 +author = Delio Brignoli diff --git a/ubus/setup.py b/ubus/setup.py new file mode 100644 index 000000000..592002653 --- /dev/null +++ b/ubus/setup.py @@ -0,0 +1,17 @@ +import sys +# Remove current dir from sys.path, otherwise setuptools will peek up our +# module instead of system. +sys.path.pop(0) +from setuptools import setup + + +setup(name='micropython-ubus', + version='0.0.1', + description='ubus interface for MicroPython', + long_description="", + author='Delio Brignoli', + author_email='brignoli.delio@gmail.com', + maintainer='Delio Brignoli', + maintainer_email='brignoli.delio@gmail.com', + license='MIT', + packages=['ubus']) diff --git a/ubus/ubus/__init__.py b/ubus/ubus/__init__.py new file mode 100644 index 000000000..738cd025b --- /dev/null +++ b/ubus/ubus/__init__.py @@ -0,0 +1,655 @@ + +import os +import _ubus +import contextlib +import uasyncio +import logging +import functools +from . import uasyncio_utils + + +def check_error(retval): + if retval == 0: + return retval + elif retval < 0: + return os.check_error(retval) + else: + raise RuntimeError(retval) + + +_instances = {} + +def _complete_handler(deferred, _ctx, _req, _ret): + deferred.set_result(_ret) + + +def blob_len(blob_attr): + return (blob_attr.id_len & 0x00ffffff) - uct.sizeof(blob_attr) + + +def _req_data_accumulator(_list): + def _data_cb(_ctx, _req, _type, _msg): + _list.append(_msg) + return _data_cb + + +async def retry_ubus_func(peer, func, *args): + while True: + ret = func(*args) + if ret == _ubus.UBUS_STATUS_OK or peer.conn.up: + return ret + logging.info('retry_ubus_func() %s %s returned %s', func, args, ret) + await peer.conn.until_up() + + +async def call_once_conn_up(conn, func, *args): + while True: + try: + if not conn.up: + await conn.until_up() + return await func(*args) + except uasyncio.CancelledError: + continue + + +async def call_once_object_available(peer, obj, func, *args): + while True: + try: + await peer.waitfor_obj(obj) + return await func(*args) + except uasyncio.CancelledError: + continue + + +def process_recv_data(ctx, ev_loop, fd): + # process incoming messages + ctx.process_recv_data() + # re-arm poll fd + ev_loop.add_reader(fd, process_recv_data, ctx, ev_loop, fd) + + +class RemoteObjectRemoved(Exception): + pass + + +class UBusObjNotificationSubscriber: + def __init__(self, ubus_peer, obj_proxy, subscriber_obj): + self.peer = ubus_peer + self.obj_proxy = obj_proxy + self._subscribed = False + self._subscriber_obj = subscriber_obj + wq = uasyncio_utils.AFuture() + self.notification_wq = wq + self.obj_proxy._notification_wqs[wq] = wq + + async def __aenter__(self): + await self.setup() + return self + + async def __aexit__(self, exc_type, exc, tb): + self.teardown() + + async def setup(self): + return await call_once_object_available(self.peer, self.obj_proxy, self._setup) + + def teardown(self): + self.obj_proxy._notification_wqs.pop(self.notification_wq, None) + + async def _setup(self): + if self.notification_wq.cancelled or not self._subscribed: + ctx = self.peer.conn.ctx + # (re)subscribe to notification by object id + ret = ctx.register_subscriber(self._subscriber_obj) + if ret in (_ubus.UBUS_STATUS_CONNECTION_FAILED, _ubus.UBUS_STATUS_TIMEOUT, _ubus.UBUS_STATUS_NOT_FOUND): + logging.error('register_subscriber() ret %s', ret) + raise uasyncio.CancelledError + check_error(ret) + ret = ctx.subscribe(self._subscriber_obj, self.obj_proxy.obj_id) + if ret in (_ubus.UBUS_STATUS_CONNECTION_FAILED, _ubus.UBUS_STATUS_TIMEOUT, _ubus.UBUS_STATUS_NOT_FOUND): + logging.error('ctx.subscribe() ret %s', ret) + raise uasyncio.CancelledError + check_error(ret) + self.notification_wq.reset() + self._subscribed = True + + async def next_notification(self): + if not self.notification_wq.cancelled and self.notification_wq.has_result(): + return self.notification_wq.result_nowait() + # Subscribe if not subscribed + if self.notification_wq.cancelled or not self._subscribed: + await self.setup() + # Wait for next notification + return await self.notification_wq.result() + + +class UBusObjProxy: + def __init__(self, ubus_peer, obj_path, obj_id, type_id): + self.peer = ubus_peer + self.path = obj_path + self.type_id = type_id + self.obj_id = obj_id + self._subscribed = False + self._subscriber_obj = _ubus.subscriber(self._handle_obj_notification) + self._cancel_on_remove = {} + self._notification_wqs = {} + + def is_stale(self): + """Return True if the remote object was removed at some point. + + This is useful to know in advance if a remote call may wait, possibly + forever, for an object to become available again. + """ + return not self.peer.obj_cache.is_obj_cached(self) + + def update_id(self, obj_path, obj_id, type_id): + if self.path != obj_path: + raise ValueError('Identity of proxy cannot change {} -> {}'.format(self.path, obj_path)) + self.obj_id = obj_id + self.type_id = type_id + + # cache known objects by proxy_obj's reference (.id() not ubus_id) + # a proxy_obj's path is what identifies the object globally + # so when a call is placed if the object .id() is not present it means + # the object went away and we should wait for its *path* to come back + # once the path is available again the new ubus_id is assigned to the + # existing proxy_obj instance. If type_id has changed an exception is raised. + + def cancel_on_remove_nocontext(self, future): + if self.is_stale(): + future.cancel() + else: + if future not in self._cancel_on_remove: + self._cancel_on_remove[future] = future + + def cancel_on_remove_discard(self, future): + self._cancel_on_remove.pop(future, None) + + @contextlib.contextmanager + def cancel_on_remove(self, future): + try: + self.cancel_on_remove_nocontext(future) + yield + finally: + self._cancel_on_remove.pop(future, None) + + def removed(self): + for future in self._cancel_on_remove.values(): + future.cancel() + self._cancel_on_remove.clear() + # Cancelled but not removed + for future in self._notification_wqs.values(): + future.cancel() + + async def invoke_method(self, method, data=None): + try: + return await call_once_object_available(self.peer, self, self._invoke_method, method, data) + finally: + self.peer.conn.ctx.process_pending() + + async def _invoke_method(self, method, data): + conn = self.peer.conn + ctx = self.peer.conn.ctx + # FIXME: turn _complete_handler into a lambda + with uasyncio_utils.AsyncCallback(_complete_handler) as req, \ + self.cancel_on_remove(req): + try: + res = [] + _req = _ubus.request(req.cb, _req_data_accumulator(res)) + ret = ctx.invoke_async(self.obj_id, method, data, _req) + if ret in (_ubus.UBUS_STATUS_CONNECTION_FAILED, _ubus.UBUS_STATUS_TIMEOUT, _ubus.UBUS_STATUS_NOT_FOUND): + raise uasyncio.CancelledError + check_error(ret) + ctx.complete_request_async(_req) + status = await req.done() + return (status, res, -1) + except: + ctx.abort_request(_req) + raise + + # Notifications + + def _handle_obj_notification(self, _ctx, _sub, notification, msg): + for wq in self._notification_wqs: + wq.set_result((self, notification, [msg])) + + def notification_subscriber(self): + return UBusObjNotificationSubscriber(self.peer, self, self._subscriber_obj) + + +class UBusObjInstance: + def __init__(self, peer, ubus_obj_name, dispatch_func): + self.peer = peer + self._obj = _ubus.object(ubus_obj_name, dispatch_func) + + async def publish(self): + return await call_once_conn_up(self.peer.conn, self._publish) + + async def _publish(self): + ret = self.peer.conn.ctx.add_object(self._obj) + if ret in (_ubus.UBUS_STATUS_CONNECTION_FAILED, _ubus.UBUS_STATUS_TIMEOUT): + raise uasyncio.CancelledError + check_error(ret) + return ret + + async def withdraw(self): + return await call_once_conn_up(self.peer.conn, self._withdraw) + + async def _withdraw(self): + ret = self.peer.conn.ctx.remove_object(self._obj) + if ret in (_ubus.UBUS_STATUS_CONNECTION_FAILED, _ubus.UBUS_STATUS_TIMEOUT): + raise uasyncio.CancelledError + check_error(ret) + return ret + + def notify_subscribers(self, notification, msg): + ret = self.peer.conn.ctx.notify(self._obj, notification, msg) + check_error(ret) + + async def wait_request(self, payload): + pass + + async def send_reply(self, request, payload): + pass + + +class UBusObjProxyCache: + def __init__(self): + self._obj_by_path = {} + self._ubusid_by_obj = {} + self._obj_by_id = {} + + def is_obj_cached(self, proxy_obj): + return proxy_obj in self._ubusid_by_obj + + def lookup(self, path): + return self._obj_by_path.get(path, None) + + def lookup_id(self, _id): + return self._obj_by_id.get(_id, None) + + def add(self, num_id, path, obj): + assert path not in self._obj_by_path + self._obj_by_path[path] = obj + assert obj not in self._ubusid_by_obj + self._ubusid_by_obj[obj] = num_id + assert id not in self._obj_by_id + self._obj_by_id[num_id] = obj + + def remove(self, path): + obj = self._obj_by_path.get(path, None) + if obj: + del self._obj_by_path[path] + del self._ubusid_by_obj[obj] + del self._obj_by_id[obj.obj_id] + obj.removed() + return obj + + def flush(self): + for obj in self._obj_by_path.values(): + obj.removed() + self._obj_by_path.clear() + self._ubusid_by_obj.clear() + self._obj_by_id.clear() + + +class UBusConnection: + def __init__(self, socket_path, peer_disconnect_cb, obj_event_handler=None): + self.path = socket_path + self.ctx = _ubus.ctx() + self.up = False + self.established = uasyncio_utils.AFuture() + self.peer_disconnect_cb = peer_disconnect_cb + self.lost = uasyncio_utils.AsyncCallback(self._disconnect_cb) + self.lost.set_result(True) + self.obj_event_handler = obj_event_handler + self._cancel_on_disconnect = {} + + def _disconnect_cb(self, deferred, _ctx): + # Hold everyone up until we are connected again + self.up = False + deferred.set_result(True) + self.established.reset() + self.peer_disconnect_cb() + for future in self._cancel_on_disconnect.values(): + future.cancel() + self._cancel_on_disconnect.clear() + + async def until_up(self): + return await self.established.result(consume=False) + + async def until_down(self): + return await self.lost.result(consume=False) + + def try_connect(self): + ctx = self.ctx + path = self.path + ev_loop = uasyncio.get_event_loop() + try: + ret = ctx.connect(path, self.lost.cb) + check_error(ret) + if self.obj_event_handler: + ret = ctx.register_event_handler(self.obj_event_handler, 'ubus.object.add') + check_error(ret) + ret = ctx.register_event_handler(self.obj_event_handler, 'ubus.object.remove') + check_error(ret) + logging.info('connecting to {} succeded'.format(path)) + except Exception as e: + logging.error('connecting to {} failed: {}'.format(path, e)) + raise + # Setup I/O callback + ev_loop = uasyncio.get_event_loop() + fd = ctx.fileno() + ev_loop.add_reader(fd, process_recv_data, ctx, ev_loop, fd) + # Signal we are connected + self.up = True + self.established.set_result(True) + self.lost.reset() + + async def _retry_connect(self, retry_interval): + while True: + with contextlib.suppress(Exception): + self.try_connect() + break + await uasyncio.sleep(retry_interval) + return True + + async def connect_and_maintain(self, retry_interval=1): + ctx = self.ctx + path = self.path + connection_lost = self.lost.result + ev_loop = uasyncio.get_event_loop() + try: + while True: + if not self.up: + # Keep trying to reconnect until we succeed + await self._retry_connect(retry_interval) + # Wait until connection is severed + if not await connection_lost(): + continue + logging.info('disconnected from {}'.format(path)) + # Stop polling the socket's file descriptor + ev_loop.remove_reader(ctx.fileno(), process_recv_data) + finally: + with contextlib.suppress(KeyError): + uasyncio.get_event_loop().remove_reader(ctx.fileno(), process_recv_data) + if self.obj_event_handler: + ctx.unregister_event_handler(self.obj_event_handler) + ctx.shutdown() + self.established.reset() + + def cancel_on_disconnect_nocontext(self, future): + if not self.up: + future.cancel() + else: + if future not in self._cancel_on_disconnect: + self._cancel_on_disconnect[future] = future + + def cancel_on_disconnect_discard(self, future): + self._cancel_on_disconnect.pop(future, None) + + @contextlib.contextmanager + def cancel_on_disconnect(self, future): + try: + self.cancel_on_disconnect_nocontext(future) + yield + finally: + self._cancel_on_disconnect.pop(future, None) + + +class UBusEventSubscriber: + def __init__(self, peer): + self.peer = peer + self._ev_handler = _ubus.event_handler(self._ev_callback) + self.patterns = [] + self.ev_q = uasyncio_utils.AFuture() + peer.conn.cancel_on_disconnect_nocontext(self.ev_q) + + def _ev_callback(self, _ctx, _ev, ev, msg): + self.ev_q.set_result((ev, msg)) + + async def _refresh_registrations(self): + for pattern in self.patterns: + ret = self.peer.conn.ctx.register_event_handler(self._ev_handler, pattern) + if ret in (_ubus.UBUS_STATUS_CONNECTION_FAILED, _ubus.UBUS_STATUS_TIMEOUT): + raise uasyncio.CancelledError + check_error(ret) + + async def register_pattern(self, pattern): + return await call_once_conn_up(self.peer.conn, self._register_pattern, pattern) + + async def _register_pattern(self, pattern): + if pattern in self.patterns: + return + conn = self.peer.conn + conn.cancel_on_disconnect_nocontext(self.ev_q) + if self.ev_q.cancelled: + await self._refresh_registrations() + self.ev_q.reset() + conn.cancel_on_disconnect_nocontext(self.ev_q) + ret = conn.ctx.register_event_handler(self._ev_handler, pattern) + if ret in (_ubus.UBUS_STATUS_CONNECTION_FAILED, _ubus.UBUS_STATUS_TIMEOUT): + raise uasyncio.CancelledError + check_error(ret) + self.patterns.append(pattern) + + async def event(self): + return await call_once_conn_up(self.peer.conn, self._event) + + async def _event(self): + if not self.ev_q.cancelled and self.ev_q.has_result(): + return self.ev_q.result_nowait() + conn = self.peer.conn + conn.cancel_on_disconnect_nocontext(self.ev_q) + if self.ev_q.cancelled: + await self._refresh_registrations() + self.ev_q.reset() + conn.cancel_on_disconnect_nocontext(self.ev_q) + return await self.ev_q.result() + + def close(self): + self.patterns.clear() + self.peer.conn.cancel_on_disconnect_discard(self.ev_q) + ret = self.peer.conn.ctx.unregister_event_handler(self._ev_handler) + # Ignore failures to unregister to avoid errors on shutdown + if ret in (_ubus.UBUS_STATUS_CONNECTION_FAILED, _ubus.UBUS_STATUS_TIMEOUT): + return + check_error(ret) + + +class UBusPeer: + def __init__(self, socket_path): + ev = _ubus.event_handler(self._handle_obj_event) + self.conn = UBusConnection(socket_path, self._disconnect_cb, obj_event_handler=ev) + self.obj_cache = UBusObjProxyCache() + self._run_task = None + self._waiting_objs = {} + + # Peer lifecycle management + + def __enter__(self): + self.conn.try_connect() + self.run() + return self + + def __exit__(self, type, value, tb): + self.shutdown() + + async def __aenter__(self): + self.run() + await self.conn.until_up() + return self + + async def __aexit__(self, type, value, tb): + self.shutdown() + + def try_connect(self): + self.conn.try_connect() + + def run(self, connect_retry_interval=1): + ubus_connect = self.conn.connect_and_maintain(retry_interval=connect_retry_interval) + self._run_task = uasyncio.ensure_future(ubus_connect) + + def shutdown(self): + if self._run_task: + self._run_task.close() + + def _disconnect_cb(self): + # Make all circulating proxy objs stale + self.obj_cache.flush() + # Wake up anyone waiting for a path with an exception + for wq_list in self._waiting_objs.values(): + for wq in wq_list: + wq.cancel() + self._waiting_objs.clear() + + # Object lookup and caching + + def _handle_obj_event(self, _ctx, _ev, ev, msg): + attrs = dict(_ubus.blob_decode(msg)) + path = attrs['path'] + if ev == 'ubus.object.add': + self._process_obj_added(path) + elif ev == 'ubus.object.remove': + self._process_obj_removed(path) + else: + logging.error('Unexpected object event: %s', attrs) + + def _process_obj_added(self, path): + # FIXME: error on adding a wait queue when the connection is down + wait_queues = self._waiting_objs.get(path, ()) + for q in wait_queues: + q.set_result(path) + if wait_queues: + del self._waiting_objs[path] + + def _process_obj_removed(self, path): + self.obj_cache.remove(path) + + async def _lookup(self, path): + if path.endswith('*'): + raise ValueError('wildcard lookup not implemented') + lookup_cb = lambda f,r:f.set_result(r) + conn = self.conn + with uasyncio_utils.AsyncCallback(lookup_cb) as lookup_future, \ + conn.cancel_on_disconnect(lookup_future): + ret = conn.ctx.lookup(path, lookup_future.cb) + if ret == _ubus.UBUS_STATUS_CONNECTION_FAILED: + raise uasyncio.CancelledError + if ret == _ubus.UBUS_STATUS_NOT_FOUND: + return None + check_error(ret) + return await lookup_future.result() + + async def lookup(self, path, proxy_factory = UBusObjProxy): + cached = self.obj_cache.lookup(path) + if cached: + return cached + # not cached, look it up on the server + res = await call_once_conn_up(self.conn, self._lookup, path) + if not res: + return res + # create proxy object and cache it + o = proxy_factory(self, *res) + path, num_id, type_id = res + self.obj_cache.add(num_id, path, o) + return o + + # Event management + async def send_event(self, event, payload): + try: + return await call_once_conn_up(self.conn, self._send_event, event, payload) + finally: + self.conn.ctx.process_pending() + + async def _send_event(self, event, payload): + conn = self.conn + ctx = self.conn.ctx + ret = ctx.send_event(event, payload) + if ret == _ubus.UBUS_STATUS_CONNECTION_FAILED: + raise uasyncio.CancelledError + check_error(ret) + + @contextlib.contextmanager + def event_subscriber(self): + ev_sub = UBusEventSubscriber(self) + try: + yield ev_sub + finally: + ev_sub.close() + + async def waitfor_paths(self, obj_paths): + return await call_once_conn_up(self.conn, self._waitfor_paths, obj_paths) + + async def _waitfor_paths(self, obj_paths): + conn = self.conn + with uasyncio_utils.AFuture() as wq, \ + conn.cancel_on_disconnect(wq): + for path in obj_paths: + wq_set = self._waiting_objs.setdefault(path, set()) + wq_set.add(wq) + outstanding = set(obj_paths) + for path in obj_paths: + res = await self._lookup(path) + if res: + assert res[0] == path + outstanding.remove(path) + self._waiting_objs.get(path, set()).discard(wq) + while outstanding: + path = await wq.result() + outstanding.discard(path) + return obj_paths + + async def waitfor_obj(self, obj_proxy): + return await call_once_conn_up(self.conn, self._waitfor_obj, obj_proxy) + + async def _waitfor_obj(self, obj_proxy): + while not self.obj_cache.is_obj_cached(obj_proxy): + res = await self._lookup(obj_proxy.path) + if not res: + await self._waitfor_paths([obj_proxy.path]) + continue + obj_proxy.update_id(*res) + path, num_id, type_id = res + self.obj_cache.add(num_id, path, obj_proxy) + break + return obj_proxy + + # Local object creation + + def object(self, ubus_obj_name, dispatch_func): + return UBusObjInstance(self, ubus_obj_name, dispatch_func) + + # Remote method invokation is handled by the obj proxy + + +def peer(socket_path=None): + """ + >>> p.peer() + >>> try: + >>> p.startup() + >>> # do something with p + >>> finally: + >>> p.shutdown() + + or + + >>> with peer() as p: + >>> # do something with p + >>> + """ + if socket_path not in _instances: + _instances[socket_path] = UBusPeer(socket_path) + return _instances[socket_path] + + +async def connected_peer(socket_path=None): + """ + >>> try: + >>> p = await connected_peer() + >>> # do something with p + >>> finally: + >>> p.shutdown() + """ + p = peer(socket_path=socket_path).startup() + await p.conn.until_up() + return p diff --git a/ubus/ubus/uasyncio_utils.py b/ubus/ubus/uasyncio_utils.py new file mode 100644 index 000000000..00f519f0d --- /dev/null +++ b/ubus/ubus/uasyncio_utils.py @@ -0,0 +1,141 @@ + +import os +import ffi +import uasyncio +import functools +import logging +from collections.deque import deque + + +class TimeoutError(Exception): + pass + + +class ATimeout: + def __init__(self, task, timeout_seconds, exc = TimeoutError): + self._task = task + self._timeout_task = self._fire(exc) + self._timeout = timeout_seconds + + def __enter__(self): + return self.start() + + def __exit__(self, type, value, tb): + self.cancel() + + def start(self): + uasyncio.get_event_loop().call_later(self._timeout, self._timeout_task) + return self + + def cancel(self): + self._timeout_task.close() + + async def run(self): + self.start() + res = await self._task + self.cancel() + return res + + async def _fire(self, exc): + self._task.throw(exc) + + +class AFuture: + def __init__(self, coro = None): + self._coro = coro + self.cancelled = False + self.closed = False + self._result = deque() + _in, _out = os.pipe() + self._in, self._out = open(_in, 'rb'), open(_out, 'wb') + self.done = self.result + self.waiting = 0 + self.pending_bytes = 0 + + def __enter__(self): + return self + + def __exit__(self, type, value, tb): + self.close() + + async def task(self): + try: + res = await self._coro + self.set_result(res) + except Exception as e: + self.set_result(e) + raise + + def cancel(self): + if self.closed: + if self.waiting: + raise RuntimeError('Canceling a closed future %s with %d waiting tasks', self, self.waiting) + return + if not self.cancelled: + self.cancelled = True + self._wake_waiting() + if self._coro: + uasyncio.get_event_loop().cancel(self._coro) + + def close(self): + if self.waiting: + logging.warning('Closing %s with %d waiting tasks', self, self.waiting) + if not self.closed: + self.cancel() + self.reset() + self._in.close() + self._out.close() + self.closed = True + + def has_result(self): + return len(self._result) > 0 + + def reset(self): + if self.closed: + raise ValueError('reset() on a closed future') + self._in.read(self.pending_bytes) + self.pending_bytes = 0 + self.cancelled = False + # No clear() in upy's ucollections implementation! + while len(self._result): + self._result.pop() + + def _wake_waiting(self): + self.pending_bytes += 1 + self._out.write('-') + + + def set_result(self, result): + #if self.cancelled or self.closed: + # raise RuntimeError('set_result() with cancelled %s and closed %s'.format(self.cancelled, self.closed)) + self._result.append(result) + # Signal result is available + self._wake_waiting() + + def result_nowait(self, consume=True, flush=False): + # No subscripting in upy's ucollections implementation, so need to pop + # then append :( + res = self._result.popleft() + if flush: + self.reset() + elif not consume: + self._result.appendleft(res) + return res + + async def result(self, consume=True, flush=False): + self.waiting += 1 + while not len(self._result) and not self.cancelled: + yield uasyncio.IORead(self._in) + if consume: + self._in.read(1) + self.pending_bytes -= 1 + self.waiting -= 1 + if self.cancelled: + raise uasyncio.CancelledError + return self.result_nowait(consume=consume, flush=flush) + + +class AsyncCallback(AFuture): + def __init__(self, _callable): + super().__init__() + self.cb = functools.partial(_callable, self) diff --git a/unittest/metadata.txt b/unittest/metadata.txt index 896a97819..f3c23ccee 100644 --- a/unittest/metadata.txt +++ b/unittest/metadata.txt @@ -1,3 +1,3 @@ srctype = micropython-lib type = module -version = 0.3.1 +version = 0.3.2 diff --git a/unittest/setup.py b/unittest/setup.py index f328b2899..2b9990eb6 100644 --- a/unittest/setup.py +++ b/unittest/setup.py @@ -7,7 +7,7 @@ import sdist_upip setup(name='micropython-unittest', - version='0.3.1', + version='0.3.2', description='unittest module for MicroPython', long_description="This is a module reimplemented specifically for MicroPython standard library,\nwith efficient and lean design in mind. Note that this module is likely work\nin progress and likely supports just a subset of CPython's corresponding\nmodule. Please help with the development if you are interested in this\nmodule.", url='https://github.com/micropython/micropython-lib', diff --git a/unittest/unittest.py b/unittest/unittest.py index 4d5109380..0361c8648 100644 --- a/unittest/unittest.py +++ b/unittest/unittest.py @@ -1,3 +1,6 @@ +import sys + + class SkipTest(Exception): pass @@ -217,3 +220,5 @@ def test_cases(m): suite.addTest(c) runner = TestRunner() result = runner.run(suite) + # Terminate with non zero return code in case of failures + sys.exit(result.failuresNum > 0) diff --git a/upip/bootstrap_upip.sh b/upip/bootstrap_upip.sh index 35446b9f2..9692f450c 100755 --- a/upip/bootstrap_upip.sh +++ b/upip/bootstrap_upip.sh @@ -9,7 +9,7 @@ fi # Remove any stale old version rm -rf micropython-upip-* -wget -nd -r -l1 https://pypi.python.org/pypi/micropython-upip/ --accept-regex ".*pypi.python.org/packages/source/.*.gz" --reject=html +wget -nd -rH -l1 -D files.pythonhosted.org https://pypi.org/project/micropython-upip/ --reject=html tar xfz micropython-upip-*.tar.gz mkdir -p ~/.micropython/lib/ diff --git a/upip/metadata.txt b/upip/metadata.txt index 690ecf776..95d03c03d 100644 --- a/upip/metadata.txt +++ b/upip/metadata.txt @@ -1,6 +1,6 @@ srctype = micropython-lib type = module -version = 1.2.3 +version = 1.2.4 author = Paul Sokolovsky extra_modules = upip_utarfile desc = Simple package manager for MicroPython. diff --git a/upip/setup.py b/upip/setup.py index 59b8fdc8c..3fb55af9e 100644 --- a/upip/setup.py +++ b/upip/setup.py @@ -7,7 +7,7 @@ import sdist_upip setup(name='micropython-upip', - version='1.2.3', + version='1.2.4', description='Simple package manager for MicroPython.', long_description='Simple self-hosted package manager for MicroPython (requires usocket, ussl, uzlib, uctypes builtin modules). Compatible only with packages without custom setup.py code.', url='https://github.com/micropython/micropython-lib', diff --git a/upip/upip.py b/upip/upip.py index 329d3d2e2..a400c3174 100644 --- a/upip/upip.py +++ b/upip/upip.py @@ -156,7 +156,7 @@ def url_open(url): def get_pkg_metadata(name): - f = url_open("https://pypi.python.org/pypi/%s/json" % name) + f = url_open("https://pypi.org/pypi/%s/json" % name) try: return json.load(f) finally: