feat: indie status page MVP -- FastAPI + SQLite
- 8 DB models (services, incidents, monitors, subscribers, etc.) - Full CRUD API for services, incidents, monitors - Public status page with live data - Incident detail page with timeline - API key authentication - Uptime monitoring scheduler - 13 tests passing - TECHNICAL_DESIGN.md with full spec
This commit is contained in:
IndieStatusBot
commit
902133edd3
4655 changed files with 1342691 additions and 0 deletions
457
venv/lib/python3.11/site-packages/mypy/ipc.py
Normal file
457
venv/lib/python3.11/site-packages/mypy/ipc.py
Normal file
|
|
@ -0,0 +1,457 @@
|
|||
"""Cross-platform abstractions for inter-process communication
|
||||
|
||||
On Unix, this uses AF_UNIX sockets.
|
||||
On Windows, this uses NamedPipes.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable
|
||||
from select import select
|
||||
from types import TracebackType
|
||||
from typing import Final
|
||||
from typing_extensions import Self
|
||||
|
||||
from librt.base64 import urlsafe_b64encode
|
||||
from librt.internal import ReadBuffer, WriteBuffer
|
||||
|
||||
if sys.platform == "win32":
|
||||
# This may be private, but it is needed for IPC on Windows, and is basically stable
|
||||
import _winapi
|
||||
import ctypes
|
||||
|
||||
_IPCHandle = int
|
||||
|
||||
kernel32 = ctypes.windll.kernel32
|
||||
DisconnectNamedPipe: Callable[[_IPCHandle], int] = kernel32.DisconnectNamedPipe
|
||||
FlushFileBuffers: Callable[[_IPCHandle], int] = kernel32.FlushFileBuffers
|
||||
else:
|
||||
import socket
|
||||
|
||||
_IPCHandle = socket.socket
|
||||
|
||||
# Size of the message packed as !L, i.e. 4 bytes in network order (big-endian).
|
||||
HEADER_SIZE = 4
|
||||
|
||||
|
||||
# TODO: we should make sure consistent exceptions are raised on different platforms.
|
||||
# Currently we raise either IPCException or OSError for equivalent conditions.
|
||||
class IPCException(Exception):
|
||||
"""Exception for IPC issues."""
|
||||
|
||||
|
||||
class IPCBase:
|
||||
"""Base class for communication between the dmypy client and server.
|
||||
|
||||
This contains logic shared between the client and server, such as reading
|
||||
and writing.
|
||||
We want to be able to send multiple "messages" over a single connection and
|
||||
to be able to separate the messages. We do this by prefixing each message
|
||||
with its size in a fixed format.
|
||||
"""
|
||||
|
||||
connection: _IPCHandle
|
||||
|
||||
def __init__(self, name: str, timeout: float | None) -> None:
|
||||
self.name = name
|
||||
self.timeout = timeout
|
||||
self.message_size: int | None = None
|
||||
self.buffer = bytearray()
|
||||
|
||||
def frame_from_buffer(self) -> bytes | None:
|
||||
"""Return a full frame from the bytes we have in the buffer."""
|
||||
size = len(self.buffer)
|
||||
if size < HEADER_SIZE:
|
||||
return None
|
||||
if self.message_size is None:
|
||||
self.message_size = struct.unpack("!L", self.buffer[:HEADER_SIZE])[0]
|
||||
if size < self.message_size + HEADER_SIZE:
|
||||
return None
|
||||
# We have a full frame, avoid extra copy in case we get a large frame.
|
||||
bdata = memoryview(self.buffer)[HEADER_SIZE : HEADER_SIZE + self.message_size]
|
||||
self.buffer = self.buffer[HEADER_SIZE + self.message_size :]
|
||||
self.message_size = None
|
||||
return bytes(bdata)
|
||||
|
||||
def read(self, size: int = 100000) -> str:
|
||||
return self.read_bytes(size).decode("utf-8")
|
||||
|
||||
def read_bytes(self, size: int = 100000) -> bytes:
|
||||
"""Read bytes from an IPC connection until we have a full frame."""
|
||||
if sys.platform == "win32":
|
||||
while True:
|
||||
# Check if we already have a message in the buffer before
|
||||
# receiving any more data from the socket.
|
||||
bdata = self.frame_from_buffer()
|
||||
if bdata is not None:
|
||||
break
|
||||
|
||||
# Receive more data into the buffer.
|
||||
ov, err = _winapi.ReadFile(self.connection, size, overlapped=True)
|
||||
try:
|
||||
if err == _winapi.ERROR_IO_PENDING:
|
||||
timeout = int(self.timeout * 1000) if self.timeout else _winapi.INFINITE
|
||||
res = _winapi.WaitForSingleObject(ov.event, timeout)
|
||||
if res != _winapi.WAIT_OBJECT_0:
|
||||
raise IPCException(f"Bad result from I/O wait: {res}")
|
||||
except BaseException:
|
||||
ov.cancel()
|
||||
raise
|
||||
_, err = ov.GetOverlappedResult(True)
|
||||
more = ov.getbuffer()
|
||||
if more:
|
||||
self.buffer.extend(more)
|
||||
bdata = self.frame_from_buffer()
|
||||
if bdata is not None:
|
||||
break
|
||||
if err == 0:
|
||||
# we are done!
|
||||
break
|
||||
elif err == _winapi.ERROR_MORE_DATA:
|
||||
# read again
|
||||
continue
|
||||
elif err == _winapi.ERROR_OPERATION_ABORTED:
|
||||
raise IPCException("ReadFile operation aborted.")
|
||||
else:
|
||||
while True:
|
||||
# Check if we already have a message in the buffer before
|
||||
# receiving any more data from the socket.
|
||||
bdata = self.frame_from_buffer()
|
||||
if bdata is not None:
|
||||
break
|
||||
|
||||
# Receive more data into the buffer.
|
||||
more = self.connection.recv(size)
|
||||
if not more:
|
||||
# Connection closed
|
||||
break
|
||||
self.buffer.extend(more)
|
||||
|
||||
if not bdata:
|
||||
# Socket was empty, and we didn't get any frame.
|
||||
# This should only happen if the socket was closed.
|
||||
return b""
|
||||
return bdata
|
||||
|
||||
def write(self, data: str) -> None:
|
||||
self.write_bytes(data.encode("utf-8"))
|
||||
|
||||
def write_bytes(self, data: bytes) -> None:
|
||||
"""Write to an IPC connection."""
|
||||
|
||||
# Frame the data by adding fixed size header.
|
||||
encoded_data = struct.pack("!L", len(data)) + data
|
||||
|
||||
if sys.platform == "win32":
|
||||
try:
|
||||
ov, err = _winapi.WriteFile(self.connection, encoded_data, overlapped=True)
|
||||
try:
|
||||
if err == _winapi.ERROR_IO_PENDING:
|
||||
timeout = int(self.timeout * 1000) if self.timeout else _winapi.INFINITE
|
||||
res = _winapi.WaitForSingleObject(ov.event, timeout)
|
||||
if res != _winapi.WAIT_OBJECT_0:
|
||||
raise IPCException(f"Bad result from I/O wait: {res}")
|
||||
elif err != 0:
|
||||
raise IPCException(f"Failed writing to pipe with error: {err}")
|
||||
except BaseException:
|
||||
ov.cancel()
|
||||
raise
|
||||
bytes_written, err = ov.GetOverlappedResult(True)
|
||||
assert err == 0, err
|
||||
assert bytes_written == len(encoded_data)
|
||||
except OSError as e:
|
||||
raise IPCException(f"Failed to write with error: {e.winerror}") from e
|
||||
else:
|
||||
self.connection.sendall(encoded_data)
|
||||
|
||||
def close(self) -> None:
|
||||
if sys.platform == "win32":
|
||||
if self.connection != _winapi.NULL:
|
||||
_winapi.CloseHandle(self.connection)
|
||||
else:
|
||||
self.connection.close()
|
||||
|
||||
|
||||
class IPCClient(IPCBase):
|
||||
"""The client side of an IPC connection."""
|
||||
|
||||
def __init__(self, name: str, timeout: float | None) -> None:
|
||||
super().__init__(name, timeout)
|
||||
if sys.platform == "win32":
|
||||
timeout = int(self.timeout * 1000) if self.timeout else _winapi.NMPWAIT_WAIT_FOREVER
|
||||
try:
|
||||
_winapi.WaitNamedPipe(self.name, timeout)
|
||||
except FileNotFoundError as e:
|
||||
raise IPCException(f"The NamedPipe at {self.name} was not found.") from e
|
||||
except OSError as e:
|
||||
if e.winerror == _winapi.ERROR_SEM_TIMEOUT:
|
||||
raise IPCException("Timed out waiting for connection.") from e
|
||||
else:
|
||||
raise
|
||||
try:
|
||||
self.connection = _winapi.CreateFile(
|
||||
self.name,
|
||||
_winapi.GENERIC_READ | _winapi.GENERIC_WRITE,
|
||||
0,
|
||||
_winapi.NULL,
|
||||
_winapi.OPEN_EXISTING,
|
||||
_winapi.FILE_FLAG_OVERLAPPED,
|
||||
_winapi.NULL,
|
||||
)
|
||||
except OSError as e:
|
||||
if e.winerror == _winapi.ERROR_PIPE_BUSY:
|
||||
raise IPCException("The connection is busy.") from e
|
||||
else:
|
||||
raise
|
||||
_winapi.SetNamedPipeHandleState(
|
||||
self.connection, _winapi.PIPE_READMODE_MESSAGE, None, None
|
||||
)
|
||||
else:
|
||||
self.connection = socket.socket(socket.AF_UNIX)
|
||||
self.connection.settimeout(timeout)
|
||||
self.connection.connect(name)
|
||||
|
||||
def __enter__(self) -> IPCClient:
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_ty: type[BaseException] | None = None,
|
||||
exc_val: BaseException | None = None,
|
||||
exc_tb: TracebackType | None = None,
|
||||
) -> None:
|
||||
self.close()
|
||||
|
||||
|
||||
class IPCServer(IPCBase):
|
||||
BUFFER_SIZE: Final = 2**16
|
||||
|
||||
def __init__(self, name: str, timeout: float | None = None) -> None:
|
||||
if sys.platform == "win32":
|
||||
name = r"\\.\pipe\{}-{}.pipe".format(name, urlsafe_b64encode(os.urandom(6)).decode())
|
||||
else:
|
||||
name = f"{name}.sock"
|
||||
super().__init__(name, timeout)
|
||||
if sys.platform == "win32":
|
||||
self.connection = _winapi.CreateNamedPipe(
|
||||
self.name,
|
||||
_winapi.PIPE_ACCESS_DUPLEX
|
||||
| _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE
|
||||
| _winapi.FILE_FLAG_OVERLAPPED,
|
||||
_winapi.PIPE_READMODE_MESSAGE
|
||||
| _winapi.PIPE_TYPE_MESSAGE
|
||||
| _winapi.PIPE_WAIT
|
||||
| 0x8, # PIPE_REJECT_REMOTE_CLIENTS
|
||||
1, # one instance
|
||||
self.BUFFER_SIZE,
|
||||
self.BUFFER_SIZE,
|
||||
_winapi.NMPWAIT_WAIT_FOREVER,
|
||||
0, # Use default security descriptor
|
||||
)
|
||||
if self.connection == -1: # INVALID_HANDLE_VALUE
|
||||
err = _winapi.GetLastError()
|
||||
raise IPCException(f"Invalid handle to pipe: {err}")
|
||||
else:
|
||||
self.sock_directory = tempfile.mkdtemp()
|
||||
sockfile = os.path.join(self.sock_directory, self.name)
|
||||
self.sock = socket.socket(socket.AF_UNIX)
|
||||
self.sock.bind(sockfile)
|
||||
self.sock.listen(1)
|
||||
if timeout is not None:
|
||||
self.sock.settimeout(timeout)
|
||||
|
||||
def __enter__(self) -> IPCServer:
|
||||
if sys.platform == "win32":
|
||||
# NOTE: It is theoretically possible that this will hang forever if the
|
||||
# client never connects, though this can be "solved" by killing the server
|
||||
try:
|
||||
ov = _winapi.ConnectNamedPipe(self.connection, overlapped=True)
|
||||
except OSError as e:
|
||||
# Don't raise if the client already exists, or the client already connected
|
||||
if e.winerror not in (_winapi.ERROR_PIPE_CONNECTED, _winapi.ERROR_NO_DATA):
|
||||
raise
|
||||
else:
|
||||
try:
|
||||
timeout = int(self.timeout * 1000) if self.timeout else _winapi.INFINITE
|
||||
res = _winapi.WaitForSingleObject(ov.event, timeout)
|
||||
assert res == _winapi.WAIT_OBJECT_0
|
||||
except BaseException:
|
||||
ov.cancel()
|
||||
_winapi.CloseHandle(self.connection)
|
||||
raise
|
||||
_, err = ov.GetOverlappedResult(True)
|
||||
assert err == 0
|
||||
else:
|
||||
try:
|
||||
self.connection, _ = self.sock.accept()
|
||||
except TimeoutError as e:
|
||||
raise IPCException("The socket timed out") from e
|
||||
return self
|
||||
|
||||
def __exit__(
|
||||
self,
|
||||
exc_ty: type[BaseException] | None = None,
|
||||
exc_val: BaseException | None = None,
|
||||
exc_tb: TracebackType | None = None,
|
||||
) -> None:
|
||||
if sys.platform == "win32":
|
||||
try:
|
||||
# Wait for the client to finish reading the last write before disconnecting
|
||||
if not FlushFileBuffers(self.connection):
|
||||
raise IPCException(
|
||||
"Failed to flush NamedPipe buffer, maybe the client hung up?"
|
||||
)
|
||||
finally:
|
||||
DisconnectNamedPipe(self.connection)
|
||||
else:
|
||||
self.close()
|
||||
|
||||
def cleanup(self) -> None:
|
||||
if sys.platform == "win32":
|
||||
self.close()
|
||||
else:
|
||||
shutil.rmtree(self.sock_directory)
|
||||
|
||||
@property
|
||||
def connection_name(self) -> str:
|
||||
if sys.platform == "win32":
|
||||
return self.name
|
||||
elif sys.platform == "gnu0":
|
||||
# GNU/Hurd returns empty string from getsockname()
|
||||
# for AF_UNIX sockets
|
||||
return os.path.join(self.sock_directory, self.name)
|
||||
else:
|
||||
name = self.sock.getsockname()
|
||||
assert isinstance(name, str)
|
||||
return name
|
||||
|
||||
|
||||
class BadStatus(Exception):
|
||||
"""Exception raised when there is something wrong with the status file.
|
||||
|
||||
For example:
|
||||
- No status file found
|
||||
- Status file malformed
|
||||
- Process whose pid is in the status file does not exist
|
||||
"""
|
||||
|
||||
|
||||
def read_status(status_file: str) -> dict[str, object]:
|
||||
"""Read status file.
|
||||
|
||||
Raise BadStatus if the status file doesn't exist or contains
|
||||
invalid JSON or the JSON is not a dict.
|
||||
"""
|
||||
if not os.path.isfile(status_file):
|
||||
raise BadStatus("No status file found")
|
||||
with open(status_file) as f:
|
||||
try:
|
||||
data = json.load(f)
|
||||
except Exception as e:
|
||||
raise BadStatus(f"Malformed status file: {str(e)}") from e
|
||||
if not isinstance(data, dict):
|
||||
raise BadStatus(f"Invalid status file (not a dict): {data}")
|
||||
return data
|
||||
|
||||
|
||||
def ready_to_read(conns: list[IPCClient], timeout: float | None = None) -> list[int]:
|
||||
"""Wait until some connections are readable.
|
||||
|
||||
Return index of each readable connection in the original list.
|
||||
"""
|
||||
if sys.platform == "win32":
|
||||
# Windows doesn't support select() on named pipes. Instead, start an overlapped
|
||||
# ReadFile on each pipe (which internally creates an event via CreateEventW),
|
||||
# then WaitForMultipleObjects on those events for efficient OS-level waiting.
|
||||
# Any data consumed by the probe reads is stored into each connection's buffer
|
||||
# so the subsequent read_bytes() call will find it via frame_from_buffer().
|
||||
WAIT_FAILED = 0xFFFFFFFF
|
||||
pending: list[tuple[int, _winapi.Overlapped]] = []
|
||||
events: list[int] = []
|
||||
ready: list[int] = []
|
||||
|
||||
for i, conn in enumerate(conns):
|
||||
try:
|
||||
ov, err = _winapi.ReadFile(conn.connection, 1, overlapped=True)
|
||||
except OSError:
|
||||
# Broken/closed pipe. Mimic Linux behavior here, caller will get
|
||||
# the exception when trying to read from this socket.
|
||||
ready.append(i)
|
||||
continue
|
||||
if err == _winapi.ERROR_IO_PENDING:
|
||||
events.append(ov.event)
|
||||
pending.append((i, ov))
|
||||
else:
|
||||
# Data was immediately available (err == 0 or ERROR_MORE_DATA)
|
||||
_, err = ov.GetOverlappedResult(True)
|
||||
data = ov.getbuffer()
|
||||
if data:
|
||||
conn.buffer.extend(data)
|
||||
ready.append(i)
|
||||
|
||||
# Wait only if nothing is immediately ready and there are pending operations
|
||||
if not ready and events:
|
||||
timeout_ms = int(timeout * 1000) if timeout is not None else _winapi.INFINITE
|
||||
res = _winapi.WaitForMultipleObjects(events, False, timeout_ms)
|
||||
if res == WAIT_FAILED:
|
||||
for _, ov in pending:
|
||||
ov.cancel()
|
||||
raise IPCException(f"Failed to wait for connections: {_winapi.GetLastError()}")
|
||||
|
||||
# Check which pending operations completed, cancel the rest
|
||||
for i, ov in pending:
|
||||
if _winapi.WaitForSingleObject(ov.event, 0) == _winapi.WAIT_OBJECT_0:
|
||||
_, err = ov.GetOverlappedResult(True)
|
||||
data = ov.getbuffer()
|
||||
if data:
|
||||
conns[i].buffer.extend(data)
|
||||
ready.append(i)
|
||||
else:
|
||||
ov.cancel()
|
||||
|
||||
return ready
|
||||
|
||||
else:
|
||||
connections = [conn.connection for conn in conns]
|
||||
ready, _, _ = select(connections, [], [], timeout)
|
||||
return [connections.index(r) for r in ready]
|
||||
|
||||
|
||||
def send(connection: IPCBase, data: IPCMessage) -> None:
|
||||
"""Send data to a connection encoded and framed.
|
||||
|
||||
The data must be a non-abstract IPCMessage. We assume that a single send call is a
|
||||
single frame to be sent.
|
||||
"""
|
||||
buf = WriteBuffer()
|
||||
data.write(buf)
|
||||
connection.write_bytes(buf.getvalue())
|
||||
|
||||
|
||||
def receive(connection: IPCBase) -> ReadBuffer:
|
||||
"""Receive single encoded IPCMessage frame from a connection.
|
||||
|
||||
Raise OSError if the data received is not valid.
|
||||
"""
|
||||
bdata = connection.read_bytes()
|
||||
if not bdata:
|
||||
raise OSError("No data received")
|
||||
return ReadBuffer(bdata)
|
||||
|
||||
|
||||
class IPCMessage:
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def read(cls, buf: ReadBuffer) -> Self:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def write(self, buf: WriteBuffer) -> None:
|
||||
raise NotImplementedError
|
||||
Loading…
Add table
Add a link
Reference in a new issue