472 lines
17 KiB
Python
472 lines
17 KiB
Python
|
import asyncio
|
||
|
|
import contextlib
|
||
|
|
|
||
|
|
from concurrent.futures import ALL_COMPLETED
|
||
|
|
from async_timeout import timeout as async_timeout
|
||
|
|
|
||
|
|
from ..log import sentinel_logger
|
||
|
|
from ..pubsub import Receiver
|
||
|
|
from ..pool import create_pool, ConnectionsPool
|
||
|
|
from ..errors import (
|
||
|
|
MasterNotFoundError,
|
||
|
|
SlaveNotFoundError,
|
||
|
|
PoolClosedError,
|
||
|
|
RedisError,
|
||
|
|
MasterReplyError,
|
||
|
|
SlaveReplyError,
|
||
|
|
)
|
||
|
|
from ..util import CloseEvent
|
||
|
|
|
||
|
|
|
||
|
|
# Address marker for discovery
|
||
|
|
_NON_DISCOVERED = object()
|
||
|
|
|
||
|
|
_logger = sentinel_logger.getChild('monitor')
|
||
|
|
|
||
|
|
|
||
|
|
async def create_sentinel_pool(sentinels, *, db=None, password=None,
|
||
|
|
encoding=None, minsize=1, maxsize=10,
|
||
|
|
ssl=None, parser=None, timeout=0.2, loop=None):
|
||
|
|
"""Create SentinelPool."""
|
||
|
|
# FIXME: revise default timeout value
|
||
|
|
assert isinstance(sentinels, (list, tuple)), sentinels
|
||
|
|
# TODO: deprecation note
|
||
|
|
# if loop is None:
|
||
|
|
# loop = asyncio.get_event_loop()
|
||
|
|
|
||
|
|
pool = SentinelPool(sentinels, db=db,
|
||
|
|
password=password,
|
||
|
|
ssl=ssl,
|
||
|
|
encoding=encoding,
|
||
|
|
parser=parser,
|
||
|
|
minsize=minsize,
|
||
|
|
maxsize=maxsize,
|
||
|
|
timeout=timeout,
|
||
|
|
loop=loop)
|
||
|
|
await pool.discover()
|
||
|
|
return pool
|
||
|
|
|
||
|
|
|
||
|
|
class SentinelPool:
|
||
|
|
"""Sentinel connections pool.
|
||
|
|
|
||
|
|
Holds connection pools to known and discovered (TBD) Sentinels
|
||
|
|
as well as services' connections.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, sentinels, *, db=None, password=None, ssl=None,
|
||
|
|
encoding=None, parser=None, minsize, maxsize, timeout,
|
||
|
|
loop=None):
|
||
|
|
# TODO: deprecation note
|
||
|
|
# if loop is None:
|
||
|
|
# loop = asyncio.get_event_loop()
|
||
|
|
# TODO: add connection/discover timeouts;
|
||
|
|
# and what to do if no master is found:
|
||
|
|
# (raise error or try forever or try until timeout)
|
||
|
|
|
||
|
|
# XXX: _sentinels is unordered
|
||
|
|
self._sentinels = set(sentinels)
|
||
|
|
self._timeout = timeout
|
||
|
|
self._pools = [] # list of sentinel pools
|
||
|
|
self._masters = {}
|
||
|
|
self._slaves = {}
|
||
|
|
self._parser_class = parser
|
||
|
|
self._redis_db = db
|
||
|
|
self._redis_password = password
|
||
|
|
self._redis_ssl = ssl
|
||
|
|
self._redis_encoding = encoding
|
||
|
|
self._redis_minsize = minsize
|
||
|
|
self._redis_maxsize = maxsize
|
||
|
|
self._close_state = CloseEvent(self._do_close)
|
||
|
|
self._close_waiter = None
|
||
|
|
self._monitor = monitor = Receiver()
|
||
|
|
|
||
|
|
async def echo_events():
|
||
|
|
try:
|
||
|
|
while await monitor.wait_message():
|
||
|
|
_, (ev, data) = await monitor.get(encoding='utf-8')
|
||
|
|
ev = ev.decode('utf-8')
|
||
|
|
_logger.debug("%s: %s", ev, data)
|
||
|
|
if ev in ('+odown',):
|
||
|
|
typ, name, *tail = data.split(' ')
|
||
|
|
if typ == 'master':
|
||
|
|
self._need_rediscover(name)
|
||
|
|
# TODO: parse messages;
|
||
|
|
# watch +new-epoch which signals `failover in progres`
|
||
|
|
# freeze reconnection
|
||
|
|
# wait / discover new master (find proper way)
|
||
|
|
# unfreeze reconnection
|
||
|
|
#
|
||
|
|
# discover master in default way
|
||
|
|
# get-master-addr...
|
||
|
|
# connnect
|
||
|
|
# role
|
||
|
|
# etc...
|
||
|
|
except asyncio.CancelledError:
|
||
|
|
pass
|
||
|
|
self._monitor_task = asyncio.ensure_future(echo_events())
|
||
|
|
|
||
|
|
@property
|
||
|
|
def discover_timeout(self):
|
||
|
|
"""Timeout (seconds) for Redis/Sentinel command calls during
|
||
|
|
master/slave address discovery.
|
||
|
|
"""
|
||
|
|
return self._timeout
|
||
|
|
|
||
|
|
def master_for(self, service):
|
||
|
|
"""Returns wrapper to master's pool for requested service."""
|
||
|
|
# TODO: make it coroutine and connect minsize connections
|
||
|
|
if service not in self._masters:
|
||
|
|
self._masters[service] = ManagedPool(
|
||
|
|
self, service, is_master=True,
|
||
|
|
db=self._redis_db,
|
||
|
|
password=self._redis_password,
|
||
|
|
encoding=self._redis_encoding,
|
||
|
|
minsize=self._redis_minsize,
|
||
|
|
maxsize=self._redis_maxsize,
|
||
|
|
ssl=self._redis_ssl,
|
||
|
|
parser=self._parser_class,
|
||
|
|
)
|
||
|
|
return self._masters[service]
|
||
|
|
|
||
|
|
def slave_for(self, service):
|
||
|
|
"""Returns wrapper to slave's pool for requested service."""
|
||
|
|
# TODO: make it coroutine and connect minsize connections
|
||
|
|
if service not in self._slaves:
|
||
|
|
self._slaves[service] = ManagedPool(
|
||
|
|
self, service, is_master=False,
|
||
|
|
db=self._redis_db,
|
||
|
|
password=self._redis_password,
|
||
|
|
encoding=self._redis_encoding,
|
||
|
|
minsize=self._redis_minsize,
|
||
|
|
maxsize=self._redis_maxsize,
|
||
|
|
ssl=self._redis_ssl,
|
||
|
|
parser=self._parser_class,
|
||
|
|
)
|
||
|
|
return self._slaves[service]
|
||
|
|
|
||
|
|
def execute(self, command, *args, **kwargs):
|
||
|
|
"""Execute sentinel command."""
|
||
|
|
# TODO: choose pool
|
||
|
|
# kwargs can be used to control which sentinel to use
|
||
|
|
if self.closed:
|
||
|
|
raise PoolClosedError("Sentinel pool is closed")
|
||
|
|
for pool in self._pools:
|
||
|
|
return pool.execute(command, *args, **kwargs)
|
||
|
|
# how to handle errors and pick other pool?
|
||
|
|
# is the only way to make it coroutine?
|
||
|
|
|
||
|
|
@property
|
||
|
|
def closed(self):
|
||
|
|
"""True if pool is closed or closing."""
|
||
|
|
return self._close_state.is_set()
|
||
|
|
|
||
|
|
def close(self):
|
||
|
|
"""Close all controlled connections (both sentinel and redis)."""
|
||
|
|
if not self._close_state.is_set():
|
||
|
|
self._close_state.set()
|
||
|
|
|
||
|
|
async def _do_close(self):
|
||
|
|
# TODO: lock
|
||
|
|
tasks = []
|
||
|
|
task, self._monitor_task = self._monitor_task, None
|
||
|
|
task.cancel()
|
||
|
|
tasks.append(task)
|
||
|
|
while self._pools:
|
||
|
|
pool = self._pools.pop(0)
|
||
|
|
pool.close()
|
||
|
|
tasks.append(pool.wait_closed())
|
||
|
|
while self._masters:
|
||
|
|
_, pool = self._masters.popitem()
|
||
|
|
pool.close()
|
||
|
|
tasks.append(pool.wait_closed())
|
||
|
|
while self._slaves:
|
||
|
|
_, pool = self._slaves.popitem()
|
||
|
|
pool.close()
|
||
|
|
tasks.append(pool.wait_closed())
|
||
|
|
await asyncio.gather(*tasks)
|
||
|
|
|
||
|
|
async def wait_closed(self):
|
||
|
|
"""Wait until pool gets closed."""
|
||
|
|
await self._close_state.wait()
|
||
|
|
|
||
|
|
async def discover(self, timeout=None): # TODO: better name?
|
||
|
|
"""Discover sentinels and all monitored services within given timeout.
|
||
|
|
|
||
|
|
If no sentinels discovered within timeout: TimeoutError is raised.
|
||
|
|
If some sentinels were discovered but not all — it is ok.
|
||
|
|
If not all monitored services (masters/slaves) discovered
|
||
|
|
(or connections established) — it is ok.
|
||
|
|
TBD: what if some sentinels/services unreachable;
|
||
|
|
"""
|
||
|
|
# TODO: check not closed
|
||
|
|
# TODO: discovery must be done with some customizable timeout.
|
||
|
|
if timeout is None:
|
||
|
|
timeout = self.discover_timeout
|
||
|
|
tasks = []
|
||
|
|
pools = []
|
||
|
|
for addr in self._sentinels: # iterate over unordered set
|
||
|
|
tasks.append(self._connect_sentinel(addr, timeout, pools))
|
||
|
|
done, pending = await asyncio.wait(tasks,
|
||
|
|
return_when=ALL_COMPLETED)
|
||
|
|
assert not pending, ("Expected all tasks to complete", done, pending)
|
||
|
|
|
||
|
|
for task in done:
|
||
|
|
result = task.result()
|
||
|
|
if isinstance(result, Exception):
|
||
|
|
continue # FIXME
|
||
|
|
if not pools:
|
||
|
|
raise Exception("Could not connect to any sentinel")
|
||
|
|
pools, self._pools[:] = self._pools[:], pools
|
||
|
|
# TODO: close current connections
|
||
|
|
for pool in pools:
|
||
|
|
pool.close()
|
||
|
|
await pool.wait_closed()
|
||
|
|
|
||
|
|
# TODO: discover peer sentinels
|
||
|
|
for pool in self._pools:
|
||
|
|
await pool.execute_pubsub(
|
||
|
|
b'psubscribe', self._monitor.pattern('*'))
|
||
|
|
|
||
|
|
async def _connect_sentinel(self, address, timeout, pools):
|
||
|
|
"""Try to connect to specified Sentinel returning either
|
||
|
|
connections pool or exception.
|
||
|
|
"""
|
||
|
|
try:
|
||
|
|
with async_timeout(timeout):
|
||
|
|
pool = await create_pool(
|
||
|
|
address, minsize=1, maxsize=2,
|
||
|
|
parser=self._parser_class,
|
||
|
|
)
|
||
|
|
pools.append(pool)
|
||
|
|
return pool
|
||
|
|
except asyncio.TimeoutError as err:
|
||
|
|
sentinel_logger.debug(
|
||
|
|
"Failed to connect to Sentinel(%r) within %ss timeout",
|
||
|
|
address, timeout)
|
||
|
|
return err
|
||
|
|
except Exception as err:
|
||
|
|
sentinel_logger.debug(
|
||
|
|
"Error connecting to Sentinel(%r): %r", address, err)
|
||
|
|
return err
|
||
|
|
|
||
|
|
async def discover_master(self, service, timeout):
|
||
|
|
"""Perform Master discovery for specified service."""
|
||
|
|
# TODO: get lock
|
||
|
|
idle_timeout = timeout
|
||
|
|
# FIXME: single timeout used 4 times;
|
||
|
|
# meaning discovery can take up to:
|
||
|
|
# 3 * timeout * (sentinels count)
|
||
|
|
#
|
||
|
|
# having one global timeout also can leed to
|
||
|
|
# a problem when not all sentinels are checked.
|
||
|
|
|
||
|
|
# use a copy, cause pools can change
|
||
|
|
pools = self._pools[:]
|
||
|
|
for sentinel in pools:
|
||
|
|
try:
|
||
|
|
with async_timeout(timeout):
|
||
|
|
address = await self._get_masters_address(
|
||
|
|
sentinel, service)
|
||
|
|
|
||
|
|
pool = self._masters[service]
|
||
|
|
with async_timeout(timeout), \
|
||
|
|
contextlib.ExitStack() as stack:
|
||
|
|
conn = await pool._create_new_connection(address)
|
||
|
|
stack.callback(conn.close)
|
||
|
|
await self._verify_service_role(conn, 'master')
|
||
|
|
stack.pop_all()
|
||
|
|
|
||
|
|
return conn
|
||
|
|
except asyncio.CancelledError:
|
||
|
|
# we must correctly handle CancelledError(s):
|
||
|
|
# application may be stopped or function can be cancelled
|
||
|
|
# by outer timeout, so we must stop the look up.
|
||
|
|
raise
|
||
|
|
except asyncio.TimeoutError:
|
||
|
|
continue
|
||
|
|
except DiscoverError as err:
|
||
|
|
sentinel_logger.debug("DiscoverError(%r, %s): %r",
|
||
|
|
sentinel, service, err)
|
||
|
|
await asyncio.sleep(idle_timeout)
|
||
|
|
continue
|
||
|
|
except RedisError as err:
|
||
|
|
raise MasterReplyError("Service {} error".format(service), err)
|
||
|
|
except Exception:
|
||
|
|
# TODO: clear (drop) connections to schedule reconnect
|
||
|
|
await asyncio.sleep(idle_timeout)
|
||
|
|
continue
|
||
|
|
# Otherwise
|
||
|
|
raise MasterNotFoundError("No master found for {}".format(service))
|
||
|
|
|
||
|
|
async def discover_slave(self, service, timeout, **kwargs):
|
||
|
|
"""Perform Slave discovery for specified service."""
|
||
|
|
# TODO: use kwargs to change how slaves are picked up
|
||
|
|
# (eg: round-robin, priority, random, etc)
|
||
|
|
idle_timeout = timeout
|
||
|
|
pools = self._pools[:]
|
||
|
|
for sentinel in pools:
|
||
|
|
try:
|
||
|
|
with async_timeout(timeout):
|
||
|
|
address = await self._get_slave_address(
|
||
|
|
sentinel, service) # add **kwargs
|
||
|
|
pool = self._slaves[service]
|
||
|
|
with async_timeout(timeout), \
|
||
|
|
contextlib.ExitStack() as stack:
|
||
|
|
conn = await pool._create_new_connection(address)
|
||
|
|
stack.callback(conn.close)
|
||
|
|
await self._verify_service_role(conn, 'slave')
|
||
|
|
stack.pop_all()
|
||
|
|
return conn
|
||
|
|
except asyncio.CancelledError:
|
||
|
|
raise
|
||
|
|
except asyncio.TimeoutError:
|
||
|
|
continue
|
||
|
|
except DiscoverError:
|
||
|
|
await asyncio.sleep(idle_timeout)
|
||
|
|
continue
|
||
|
|
except RedisError as err:
|
||
|
|
raise SlaveReplyError("Service {} error".format(service), err)
|
||
|
|
except Exception:
|
||
|
|
await asyncio.sleep(idle_timeout)
|
||
|
|
continue
|
||
|
|
raise SlaveNotFoundError("No slave found for {}".format(service))
|
||
|
|
|
||
|
|
async def _get_masters_address(self, sentinel, service):
|
||
|
|
# NOTE: we don't use `get-master-addr-by-name`
|
||
|
|
# as it can provide stale data so we repeat
|
||
|
|
# after redis-py and check service flags.
|
||
|
|
state = await sentinel.execute(b'sentinel', b'master',
|
||
|
|
service, encoding='utf-8')
|
||
|
|
if not state:
|
||
|
|
raise UnknownService()
|
||
|
|
state = make_dict(state)
|
||
|
|
address = state['ip'], int(state['port'])
|
||
|
|
flags = set(state['flags'].split(','))
|
||
|
|
if {'s_down', 'o_down', 'disconnected'} & flags:
|
||
|
|
raise BadState(state)
|
||
|
|
return address
|
||
|
|
|
||
|
|
async def _get_slave_address(self, sentinel, service):
|
||
|
|
# Find and return single slave address
|
||
|
|
slaves = await sentinel.execute(b'sentinel', b'slaves',
|
||
|
|
service, encoding='utf-8')
|
||
|
|
if not slaves:
|
||
|
|
raise UnknownService()
|
||
|
|
for state in map(make_dict, slaves):
|
||
|
|
address = state['ip'], int(state['port'])
|
||
|
|
flags = set(state['flags'].split(','))
|
||
|
|
if {'s_down', 'o_down', 'disconnected'} & flags:
|
||
|
|
continue
|
||
|
|
return address
|
||
|
|
raise BadState() # XXX: only last state
|
||
|
|
|
||
|
|
async def _verify_service_role(self, conn, role):
|
||
|
|
res = await conn.execute(b'role', encoding='utf-8')
|
||
|
|
if res[0] != role:
|
||
|
|
raise RoleMismatch(res)
|
||
|
|
|
||
|
|
def _need_rediscover(self, service):
|
||
|
|
sentinel_logger.debug("Must redisover service %s", service)
|
||
|
|
pool = self._masters.get(service)
|
||
|
|
if pool:
|
||
|
|
pool.need_rediscover()
|
||
|
|
pool = self._slaves.get(service)
|
||
|
|
if pool:
|
||
|
|
pool.need_rediscover()
|
||
|
|
|
||
|
|
|
||
|
|
class ManagedPool(ConnectionsPool):
|
||
|
|
|
||
|
|
def __init__(self, sentinel, service, is_master,
|
||
|
|
db=None, password=None, encoding=None, parser=None,
|
||
|
|
*, minsize, maxsize, ssl=None, loop=None):
|
||
|
|
super().__init__(_NON_DISCOVERED,
|
||
|
|
db=db, password=password, encoding=encoding,
|
||
|
|
minsize=minsize, maxsize=maxsize, ssl=ssl,
|
||
|
|
parser=parser, loop=loop)
|
||
|
|
assert self._address is _NON_DISCOVERED
|
||
|
|
self._sentinel = sentinel
|
||
|
|
self._service = service
|
||
|
|
self._is_master = is_master
|
||
|
|
# self._discover_timeout = .2
|
||
|
|
|
||
|
|
@property
|
||
|
|
def address(self):
|
||
|
|
if self._address is _NON_DISCOVERED:
|
||
|
|
return
|
||
|
|
return self._address
|
||
|
|
|
||
|
|
def get_connection(self, command, args=()):
|
||
|
|
if self._address is _NON_DISCOVERED:
|
||
|
|
return None, _NON_DISCOVERED
|
||
|
|
return super().get_connection(command, args)
|
||
|
|
|
||
|
|
async def _create_new_connection(self, address):
|
||
|
|
if address is _NON_DISCOVERED:
|
||
|
|
# Perform service discovery.
|
||
|
|
# Returns Connection or raises error if no service can be found.
|
||
|
|
await self._do_clear() # make `clear` blocking
|
||
|
|
|
||
|
|
if self._is_master:
|
||
|
|
conn = await self._sentinel.discover_master(
|
||
|
|
self._service, timeout=self._sentinel.discover_timeout)
|
||
|
|
else:
|
||
|
|
conn = await self._sentinel.discover_slave(
|
||
|
|
self._service, timeout=self._sentinel.discover_timeout)
|
||
|
|
self._address = conn.address
|
||
|
|
sentinel_logger.debug("Discoverred new address %r for %s",
|
||
|
|
conn.address, self._service)
|
||
|
|
return conn
|
||
|
|
return await super()._create_new_connection(address)
|
||
|
|
|
||
|
|
def _drop_closed(self):
|
||
|
|
diff = len(self._pool)
|
||
|
|
super()._drop_closed()
|
||
|
|
diff -= len(self._pool)
|
||
|
|
if diff:
|
||
|
|
# closed connections were in pool:
|
||
|
|
# * reset address;
|
||
|
|
# * notify sentinel pool
|
||
|
|
sentinel_logger.debug(
|
||
|
|
"Dropped %d closed connnection(s); must rediscover", diff)
|
||
|
|
self._sentinel._need_rediscover(self._service)
|
||
|
|
|
||
|
|
async def acquire(self, command=None, args=()):
|
||
|
|
if self._address is _NON_DISCOVERED:
|
||
|
|
await self.clear()
|
||
|
|
return await super().acquire(command, args)
|
||
|
|
|
||
|
|
def release(self, conn):
|
||
|
|
was_closed = conn.closed
|
||
|
|
super().release(conn)
|
||
|
|
# if connection was closed while used and not by release()
|
||
|
|
if was_closed:
|
||
|
|
sentinel_logger.debug(
|
||
|
|
"Released closed connection; must rediscover")
|
||
|
|
self._sentinel._need_rediscover(self._service)
|
||
|
|
|
||
|
|
def need_rediscover(self):
|
||
|
|
self._address = _NON_DISCOVERED
|
||
|
|
|
||
|
|
|
||
|
|
def make_dict(plain_list):
|
||
|
|
it = iter(plain_list)
|
||
|
|
return dict(zip(it, it))
|
||
|
|
|
||
|
|
|
||
|
|
class DiscoverError(Exception):
|
||
|
|
"""Internal errors for masters/slaves discovery."""
|
||
|
|
|
||
|
|
|
||
|
|
class BadState(DiscoverError):
|
||
|
|
"""Bad master's / slave's state read from sentinel."""
|
||
|
|
|
||
|
|
|
||
|
|
class UnknownService(DiscoverError):
|
||
|
|
"""Service is not monitored by specific sentinel."""
|
||
|
|
|
||
|
|
|
||
|
|
class RoleMismatch(DiscoverError):
|
||
|
|
"""Service reported to have other Role."""
|