This commit is contained in:
Untriex Programming
2021-03-17 08:57:57 +01:00
parent 339be0ccd8
commit ed6afdb5c9
3074 changed files with 423348 additions and 0 deletions

View File

@@ -0,0 +1,6 @@
"""Utilities for asyncio-friendly file handling."""
from .threadpool import open
__version__ = "0.6.0"
__all__ = ["open"]

View File

@@ -0,0 +1,8 @@
import sys
try:
from functools import singledispatch
except ImportError: # pragma: nocover
from singledispatch import singledispatch
PY_35 = sys.version_info >= (3, 5)

View File

@@ -0,0 +1,88 @@
"""Various base classes."""
from types import coroutine
from collections.abc import Coroutine
class AsyncBase:
def __init__(self, file, loop, executor):
self._file = file
self._loop = loop
self._executor = executor
def __aiter__(self):
"""We are our own iterator."""
return self
async def __anext__(self):
"""Simulate normal file iteration."""
line = await self.readline()
if line:
return line
else:
raise StopAsyncIteration
class _ContextManager(Coroutine):
__slots__ = ("_coro", "_obj")
def __init__(self, coro):
self._coro = coro
self._obj = None
def send(self, value):
return self._coro.send(value)
def throw(self, typ, val=None, tb=None):
if val is None:
return self._coro.throw(typ)
elif tb is None:
return self._coro.throw(typ, val)
else:
return self._coro.throw(typ, val, tb)
def close(self):
return self._coro.close()
@property
def gi_frame(self):
return self._coro.gi_frame
@property
def gi_running(self):
return self._coro.gi_running
@property
def gi_code(self):
return self._coro.gi_code
def __next__(self):
return self.send(None)
@coroutine
def __iter__(self):
resp = yield from self._coro
return resp
def __await__(self):
resp = yield from self._coro
return resp
async def __anext__(self):
resp = await self._coro
return resp
async def __aenter__(self):
self._obj = await self._coro
return self._obj
async def __aexit__(self, exc_type, exc, tb):
self._obj.close()
self._obj = None
class AiofilesContextManager(_ContextManager):
"""An adjusted async context manager for aiofiles."""
async def __aexit__(self, exc_type, exc_val, exc_tb):
await self._obj.close()
self._obj = None

View File

@@ -0,0 +1,25 @@
"""Async executor versions of file functions from the os module."""
import asyncio
from functools import partial, wraps
import os
def wrap(func):
@wraps(func)
async def run(*args, loop=None, executor=None, **kwargs):
if loop is None:
loop = asyncio.get_event_loop()
pfunc = partial(func, *args, **kwargs)
return await loop.run_in_executor(executor, pfunc)
return run
stat = wrap(os.stat)
rename = wrap(os.rename)
remove = wrap(os.remove)
mkdir = wrap(os.mkdir)
rmdir = wrap(os.rmdir)
if hasattr(os, "sendfile"):
sendfile = wrap(os.sendfile)

View File

@@ -0,0 +1,108 @@
"""Handle files using a thread pool executor."""
import asyncio
from types import coroutine
from io import (
FileIO,
TextIOBase,
BufferedReader,
BufferedWriter,
BufferedRandom,
)
from functools import partial, singledispatch
from .binary import AsyncBufferedIOBase, AsyncBufferedReader, AsyncFileIO
from .text import AsyncTextIOWrapper
from ..base import AiofilesContextManager
sync_open = open
__all__ = ("open",)
def open(
file,
mode="r",
buffering=-1,
encoding=None,
errors=None,
newline=None,
closefd=True,
opener=None,
*,
loop=None,
executor=None
):
return AiofilesContextManager(
_open(
file,
mode=mode,
buffering=buffering,
encoding=encoding,
errors=errors,
newline=newline,
closefd=closefd,
opener=opener,
loop=loop,
executor=executor,
)
)
@coroutine
def _open(
file,
mode="r",
buffering=-1,
encoding=None,
errors=None,
newline=None,
closefd=True,
opener=None,
*,
loop=None,
executor=None
):
"""Open an asyncio file."""
if loop is None:
loop = asyncio.get_event_loop()
cb = partial(
sync_open,
file,
mode=mode,
buffering=buffering,
encoding=encoding,
errors=errors,
newline=newline,
closefd=closefd,
opener=opener,
)
f = yield from loop.run_in_executor(executor, cb)
return wrap(f, loop=loop, executor=executor)
@singledispatch
def wrap(file, *, loop=None, executor=None):
raise TypeError("Unsupported io type: {}.".format(file))
@wrap.register(TextIOBase)
def _(file, *, loop=None, executor=None):
return AsyncTextIOWrapper(file, loop=loop, executor=executor)
@wrap.register(BufferedWriter)
def _(file, *, loop=None, executor=None):
return AsyncBufferedIOBase(file, loop=loop, executor=executor)
@wrap.register(BufferedReader)
@wrap.register(BufferedRandom)
def _(file, *, loop=None, executor=None):
return AsyncBufferedReader(file, loop=loop, executor=executor)
@wrap.register(FileIO)
def _(file, *, loop=None, executor=None):
return AsyncFileIO(file, loop, executor)

View File

@@ -0,0 +1,57 @@
from ..base import AsyncBase
from .utils import (
delegate_to_executor,
proxy_method_directly,
proxy_property_directly,
)
@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"read1",
"readinto",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"writable",
"write",
"writelines",
)
@proxy_method_directly("detach", "fileno", "readable")
@proxy_property_directly("closed", "raw", "name", "mode")
class AsyncBufferedIOBase(AsyncBase):
"""The asyncio executor version of io.BufferedWriter."""
@delegate_to_executor("peek")
class AsyncBufferedReader(AsyncBufferedIOBase):
"""The asyncio executor version of io.BufferedReader and Random."""
@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"readall",
"readinto",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"writable",
"write",
"writelines",
)
@proxy_method_directly("fileno", "readable")
@proxy_property_directly("closed", "name", "mode")
class AsyncFileIO(AsyncBase):
"""The asyncio executor version of io.FileIO."""

View File

@@ -0,0 +1,30 @@
from ..base import AsyncBase
from .utils import (
delegate_to_executor,
proxy_method_directly,
proxy_property_directly,
)
@delegate_to_executor(
"close",
"flush",
"isatty",
"read",
"readable",
"readline",
"readlines",
"seek",
"seekable",
"tell",
"truncate",
"write",
"writable",
"writelines",
)
@proxy_method_directly("detach", "fileno", "readable")
@proxy_property_directly(
"buffer", "closed", "encoding", "errors", "line_buffering", "newlines", "name", "mode"
)
class AsyncTextIOWrapper(AsyncBase):
"""The asyncio executor version of io.TextIOWrapper."""

View File

@@ -0,0 +1,52 @@
import functools
from types import coroutine
def delegate_to_executor(*attrs):
def cls_builder(cls):
for attr_name in attrs:
setattr(cls, attr_name, _make_delegate_method(attr_name))
return cls
return cls_builder
def proxy_method_directly(*attrs):
def cls_builder(cls):
for attr_name in attrs:
setattr(cls, attr_name, _make_proxy_method(attr_name))
return cls
return cls_builder
def proxy_property_directly(*attrs):
def cls_builder(cls):
for attr_name in attrs:
setattr(cls, attr_name, _make_proxy_property(attr_name))
return cls
return cls_builder
def _make_delegate_method(attr_name):
@coroutine
def method(self, *args, **kwargs):
cb = functools.partial(getattr(self._file, attr_name), *args, **kwargs)
return (yield from self._loop.run_in_executor(self._executor, cb))
return method
def _make_proxy_method(attr_name):
def method(self, *args, **kwargs):
return getattr(self._file, attr_name)(*args, **kwargs)
return method
def _make_proxy_property(attr_name):
def proxy_property(self):
return getattr(self._file, attr_name)
return property(proxy_property)