class Pool:
"""Thread-safe pool of something, created on demand.
Mandatory parameter:
- fun: function to create objects on demand, called with the creation number.
Pool size management parameters:
- max_size: maximum size of pool, 0 for unlimited.
- min_size: minimum size of pool.
Recycling parameters:
- max_use: how many times to use an object, 0 for unlimited.
- max_avail_delay: remove objects if unused for this secs, 0.0 for unlimited.
- max_using_delay: warn if object is kept for more than this time, 0.0 for no warning.
- max_using_delay_kill: kill if object is kept for more than this time, 0.0 for no killing.
- delay: use this delay secs for house keeping rounds, 0 for computed default.
the default delay computed based on previous delay.
- health_freq: check health this every round, default is 1.
Hook parameters:
- opener: hook called on object creation.
- getter: hook called on object pool extraction.
- retter: hook called on object pool return.
- closer: hook called on object destruction.
- stats: hook called to generate per-object JSON-compatible stats.
- tracer: hook called to generate debug information on an object, default is "str".
- health: hook called to check for an available object health.
Miscellaneous parameters:
- timeout: give-up waiting after this time, None for no timeout.
this is only used when running with a bounded-size pool (`max_size`).
- log_level: set logging level for local logger.
The object life cycle is the following, with the corresponding hooks:
- objects are created by calling `fun`, after which `opener` is called.
- when an object is extracted from the pool, `getter` is called.
- when an object is returned to the pool, `retter` is called.
- when an object is _borrowed_ for checking health, `health` is called.
- when an object is removed from the pool, `closer` is called.
Objects are created:
- when the number of available object is below `min_size` for some reason.
- when an object is requested, none is available, and the number of objects
is below `max_size`.
Objects are destroyed:
- when `health` returns *False* on an house keeper round.
- when they are unused for too long (`max_avail_delay`) and if the number
of objects is strictly over `min_size`.
- when they are being used for too long (over `max_using_delay_kill`).
- when they reach the number of uses limit (`max_use`).
- when `__delete__` or `shutdown` is called.
Environment:
- **PPP_WERKZEUG_WORKAROUND**: set when running with "flask --debug"
reload mode to avoid starting a useless pool housekeeping thread.
This infrastructure is not suitable for handling very short timeouts, and
will not be very precise. The using timeout kill is expensive as the object
is effectively destroyed (so for instance an underlying network connection
would be lost) and will have to be re-created, including potential
authentication costs. This is not a replacement for carefully designing and
monitoring application resource usage.
"""
@dataclass
class UseInfo:
"""Stats about pool item usage."""
uses: int
last_get: float
last_ret: float
# FIXME should use a lock?
_created: int = 0
def __init__(
self,
fun: FunHook,
max_size: int = 0,
min_size: int = 1,
timeout: float|None = None,
# recycling
max_use: int = 0,
max_avail_delay: float = 0.0,
max_using_delay: float = 0.0,
max_using_delay_kill: float = 0.0,
health_freq: int = 1,
delay: float = 0.0,
# hooks
opener: PoolHook|None = None,
getter: PoolHook|None = None,
retter: PoolHook|None = None,
closer: PoolHook|None = None,
health: HealthHook|None = None,
stats: StatsHook|None = None,
tracer: TraceHook|None = None,
log_level: int|None = None,
):
Pool._created += 1
self._id = Pool._created
# debugging
if log_level is not None:
log.setLevel(log_level)
self._debug = (log.getEffectiveLevel() == logging.DEBUG)
self._tracer = tracer or str
self._started = datetime.datetime.now()
self._started_ts = datetime.datetime.timestamp(self._started)
# objects
self._fun = fun
# statistics
self._nobjs = 0 # current number of objects managed in pool
self._nuses = 0 # cumulated number of uses (successful get)
self._ncreating = 0 # number of creation attempts
self._ncreated = 0 # number of created objects
self._nhealth = 0 # number of health calls
self._bad_health = 0 # number of bad health detected
self._nborrows = 0 # number of objects borrowed
self._nreturns = 0 # number of objects returned
self._nkilled = 0 # number of long time using kills
self._nrecycled = 0 # number of long time avail deletes
self._nwornout = 0 # number of max_use-d objects
self._ndestroys = 0 # number of objects actually destroyed
self._hc_rounds = 0 # number of health check rounds
self._hc_errors = 0 # number of heath check errors
self._hk_rounds = 0 # number of house keeper rounds
self._hk_errors = 0 # number of house keeping errors
self._hk_time = 0.0 # cumulated time spent in house keeping
self._hk_last = 0.0 # last time a house keeping round started
# pool management
self._shutdown = False
self._timeout = timeout
self._max_size = max_size
self._min_size = min_size
self._max_use = max_use # when to recycle
self._max_avail_delay = max_avail_delay
self._max_using_delay_kill = max_using_delay_kill
self._max_using_delay_warn = max_using_delay or max_using_delay_kill
self._health_freq = health_freq
if self._max_using_delay_kill and self._max_using_delay_warn > self._max_using_delay_kill:
log.warning("inconsistent max_using_delay_warn > max_using_delay_kill")
# hooks
self._opener = opener
self._getter = getter
self._retter = retter
self._closer = closer
self._stats = stats
self._health = health
# pool's content: available vs in use objects
self._avail: set[Any] = set()
self._using: set[Any] = set()
self._todel: set[Any] = set()
# keep track of usage count and last ops
self._uses: dict[Any, Pool.UseInfo] = {}
# global pool re-entrant lock to update "self" attributes
# NOTE under max_size a timeout may take effect in the next semaphore,
# the lock is only used to manage attributes, there is no timeout on it.
self._lock = threading.RLock()
self._sem: threading.Semaphore|None = None
if self._max_size:
self._sem = threading.BoundedSemaphore(self._max_size)
# start housekeeper thread if needed
if delay:
self._delay = delay
elif self._max_avail_delay or self._max_using_delay_warn:
self._delay = self._max_avail_delay
if not self._delay or \
self._max_using_delay_warn and self._delay > self._max_using_delay_warn: # fmt: skip
self._delay = self._max_using_delay_warn
self._delay /= 2.0
else:
self._delay = 60.0 if self._health else 0.0
assert not (self._health and self._delay == 0.0)
# NOTE avoid starting an empty thread under "flask --debug"
self._housekeeper: threading.Thread|None = None
werkzeug_workaround = "PPP_WERKZEUG_WORKAROUND" in os.environ
skip_thread = (werkzeug_workaround and
os.environ.get("WERKZEUG_RUN_MAIN", "false") != "true")
if not skip_thread:
if self._delay:
self._housekeeper = threading.Thread(target=self._houseKeeping, daemon=True)
self._housekeeper.start()
# try to create the minimum number of objects
# NOTE on errors we keep on running, hoping that it will work later:
# the pool attempts to be resilient to temporary server failures.
self._fill()
elif werkzeug_workaround:
log.warning("skipping housekeeper thread creation under werkzeug empty start…")
def _log_debug(self, m):
log.debug(f"{os.getpid()}:{threading.get_ident()} {m}")
def __stats_data(self, obj, now):
"""Generate stats data for obj, under lock."""
data = {}
if self._stats: # with stat hook
data["stats"] = self._stats(obj)
else: # with tracer hook
data["trace"] = self._tracer(obj)
# also add with usage data if available
if obj in self._uses:
suo = self._uses[obj]
data.update(uses=suo.uses, last_get=suo.last_get - now, last_ret=suo.last_ret - now)
return data
def stats(self):
"""Generate a JSON-compatible structure for stats."""
with self._lock:
now = self._now()
# generic info
return {
"id": self._id,
# pool configuration
"started": self._started.isoformat(),
"min_size": self._min_size,
"max_size": self._max_size,
"max_use": self._max_use,
"timeout": self._timeout,
"delay": self._delay,
"max_avail_delay": self._max_avail_delay,
"max_using_delay_kill": self._max_using_delay_kill,
"max_using_delay_warn": self._max_using_delay_warn,
"health_freq": self._health_freq,
# pool status
"now": now,
"sem": {"value": self._sem._value, "init": self._sem._initial_value} if self._sem else None, # type: ignore
"navail": len(self._avail),
"nusing": len(self._using),
"ntodel": len(self._todel),
"running": now - self._started_ts,
"rel_hk_last": self._hk_last - now,
"time_per_hk": self._hk_time / max(self._hk_rounds, 1),
"shutdown": self._shutdown,
# detailed per-object stats
"avail": [self.__stats_data(obj, now) for obj in self._avail],
"using": [self.__stats_data(obj, now) for obj in self._using],
# counts
"nobjs": self._nobjs,
"ncreated": self._ncreated,
"ncreating": self._ncreating,
"nuses": self._nuses,
"nkilled": self._nkilled,
"nrecycled": self._nrecycled,
"nwornout": self._nwornout,
"nborrows": self._nborrows,
"nreturns": self._nreturns,
"ndestroys": self._ndestroys,
"nhealth": self._nhealth,
"bad_health": self._bad_health,
"hk_rounds": self._hk_rounds,
"hk_errors": self._hk_errors,
"hc_rounds": self._hc_rounds,
"hc_errors": self._hc_errors,
}
def __str__(self):
return json.dumps(self.stats())
def _now(self) -> float:
"""Return now as a convenient float, in seconds."""
return datetime.datetime.timestamp(datetime.datetime.now())
def _hkRound(self):
"""Housekeeping round, under lock.
Objects that are scheduled for destruction are moved to `self._todel`
so as to minimize the time passed here.
"""
self._hk_rounds += 1
now = self._now()
if self._max_using_delay_warn:
# warn/kill long running objects
long_run, long_kill, long_time = 0, 0, 0.0
for obj in list(self._using):
running = now - self._uses[obj].last_get
if running >= self._max_using_delay_warn:
long_run += 1
long_time += running
if self._max_using_delay_kill and running >= self._max_using_delay_kill:
# we cannot just return the object because another thread may keep on using it.
long_kill += 1
self._nkilled += 1
self._out(obj)
self._todel.add(obj)
# killed objects where under using and sem
if self._sem: # pragma: no cover
self._sem.release()
_ = self._debug and self._log_debug(f"sem round R {self._sem._value}/{self._sem._initial_value}")
if long_run or long_kill:
delay = (long_time / long_run) if long_run else 0.0
log.warning(f"long running objects: {long_run} ({delay} seconds, {long_kill} to kill)")
if self._max_avail_delay and self._nobjs > self._min_size:
# close spurious unused for too long objects
for obj in list(self._avail):
if now - self._uses[obj].last_ret >= self._max_avail_delay:
self._nrecycled += 1
self._out(obj)
self._todel.add(obj)
# stop deleting objects if min size is reached
if self._nobjs <= self._min_size:
break
def _health_check(self):
"""Health check, not under lock, only called from the hk thread."""
assert self._health
self._hc_rounds += 1
with self._lock:
objs = list(self._avail)
# not under lock so a stuck health check won't freeze the pool
for obj in objs:
if self._borrow(obj):
healthy = True
try:
self._nhealth += 1
healthy = self._health(obj)
except Exception as e: # pragma: no cover
self._hc_errors += 1
log.error(f"health check error: {e}")
finally:
self._return(obj)
if not healthy:
log.error(f"bad health: {self._tracer(obj)}")
self._bad_health += 1
self._out(obj)
self._todel.add(obj) # unhealthy objects are just removed
# else skipping obj in use
def _houseKeeping(self):
"""Housekeeping thread."""
log.info(f"housekeeper {threading.get_ident()} running every {self._delay}")
while not self._shutdown:
time.sleep(self._delay)
self._hk_last = self._now()
_ = self._debug and self._log_debug("housekeeper: round start")
with self._lock:
# normal round is done under lock, it must be fast!
try:
_ = self._debug and log.debug(str(self))
self._hkRound()
except Exception as e: # pragma: no cover
self._hk_errors += 1
log.error(f"housekeeper round error: {e}")
# health check is done out of locking
if self._health and self._hk_rounds % self._health_freq == 0:
self._health_check()
# actual deletions
self._empty()
# possibly re-create objects
self._fill()
# update run time
round_time = self._now() - self._hk_last
self._hk_time += round_time
_ = self._debug and self._log_debug(f"housekeeper: round done ({round_time})")
def _fill(self):
"""Create new available objects to reach min_size."""
if self._min_size > self._nobjs:
# NOTE no locking here, does not matter much
tocreate = self._min_size - self._nobjs
_ = self._debug and self._log_debug(f"filling {tocreate} objects")
for _ in range(tocreate):
# acquire a token to avoid overshooting max_size
if self._sem:
if self._sem.acquire(timeout=0.0): # pragma: no cover
_ = self._debug and self._log_debug(f"sem fill A {self._sem._value}/{self._sem._initial_value}")
else: # pragma: no cover
_ = self._debug and self._log_debug("filling skipped on acquire")
break
try:
self._new()
except Exception as e: # pragma: no cover
log.error(f"new object failed: {e}")
if self._sem:
# whether it is created or not, the semaphore is released
self._sem.release()
_ = self._debug and self._log_debug(f"sem fill R {self._sem._value}/{self._sem._initial_value}")
_ = self._debug and self._log_debug(f"filling {tocreate} objects done")
def shutdown(self, delay: float = 0.0):
"""Shutdown pool (stop housekeeper, close all objects)."""
_ = self._debug and self._log_debug("shutting down pool")
self._shutdown = True
self._min_size = 0
if self._housekeeper:
self._housekeeper.join(delay)
if self._housekeeper.is_alive(): # pragma: no cover
log.warning("shutting down pool with live housekeeper")
del self._housekeeper
self._housekeeper = None # forget thread
self.__delete__()
def _empty(self):
"""Empty current todel."""
if self._todel:
_ = self._debug and self._log_debug(f"deleting {len(self._todel)} objects")
with self._lock:
destroys = list(self._todel)
self._todel.clear()
self._ndestroys += len(destroys)
for obj in destroys:
self._destroy(obj)
def __delete__(self):
"""This should be done automatically, but eventually."""
with self._lock:
if self._using: # pragma: no cover
log.warning(f"deleting in-use objects: {len(self._using)}")
for obj in list(self._using):
self._del(obj)
self._using.clear()
for obj in list(self._avail):
self._del(obj)
self._avail.clear()
self._uses.clear()
def _create(self):
"""Create a new object (low-level)."""
_ = self._debug and self._log_debug(f"creating new obj with {self._fun}")
with self._lock:
self._ncreating += 1
# this may fail
obj = self._fun(self._ncreated)
now = self._now()
obj_info = Pool.UseInfo(0, now, now)
with self._lock:
self._ncreated += 1
self._nobjs += 1
self._uses[obj] = obj_info
return obj
def _new(self):
"""Create a new available object."""
# this may fail
obj = self._create()
_ = self._debug and log.debug(f"open {self._tracer(obj)}")
# on success, the object is availble
if self._opener:
try:
self._opener(obj)
except Exception as e:
log.error(f"exception in opener: {e}")
with self._lock:
self._avail.add(obj)
return obj
def _out(self, obj):
"""Remove an object from pool."""
seen = False
with self._lock:
if obj in self._uses:
seen = True
del self._uses[obj]
if obj in self._avail:
seen = True
self._avail.remove(obj)
if obj in self._using: # pragma: no cover
# this should not happen…
seen = True
self._using.remove(obj)
if seen:
self._nobjs -= 1
else: # possible double removal?
log.warning(f"not found in pool: {self._tracer(obj)}") # pragma: no cover
def _destroy(self, obj):
"""Destroy an object."""
_ = self._debug and log.debug(f"close {self._tracer(obj)}")
if self._closer:
try:
self._closer(obj)
except Exception as e:
log.error(f"exception in closer: {e}")
del obj
def _del(self, obj):
"""Delete an object."""
self._out(obj)
self._destroy(obj)
def _borrow(self, obj):
"""Borrow an existing object.
This is a special get with does not get through getter or setter,
for internal use such as house keeping, health check…
If the object is not available, _None_ is returned, this is just best effort.
"""
if self._sem:
if self._sem.acquire(timeout=0.0): # pragma: no cover
_ = self._debug and self._log_debug(f"sem borrow A {self._sem._value}/{self._sem._initial_value}")
else: # pragma: no cover
return None
with self._lock:
if obj in self._avail:
self._avail.remove(obj)
self._using.add(obj)
self._nborrows += 1
_ = self._debug and log.debug(f"borrow {self._tracer(obj)}")
return obj
# else we failed to borrow it, so release semaphore!
if self._sem: # pragma: no cover
self._sem.release()
_ = self._debug and self._log_debug(f"sem borrow R {self._sem._value}/{self._sem._initial_value}")
return None # pragma: no cover
def _return(self, obj):
"""Return borrowed object."""
with self._lock:
_ = self._debug and log.debug(f"return {self._tracer(obj)}")
assert obj in self._using
self._using.remove(obj)
self._avail.add(obj)
self._nreturns += 1
if self._sem: # pragma: no cover
self._sem.release()
_ = self._debug and self._log_debug(f"sem return R {self._sem._value}/{self._sem._initial_value}")
def get(self, timeout=None):
"""Get a object from the pool, possibly creating one if needed."""
if self._shutdown: # pragma: no cover
raise PoolException("Pool is shutting down")
if self._sem: # ensure that we do not go over max_size
# the acquired token will be released at the end of ret()
# the semaphore acts as a gate keeper to the max_size connections
if not self._sem.acquire(timeout=timeout or self._timeout):
raise TimeOut(f"sem timeout after {timeout or self._timeout}")
_ = self._debug and self._log_debug(f"sem get A {self._sem._value}/{self._sem._initial_value}")
with self._lock:
if not self._avail:
try:
self._new()
except Exception as e: # pragma: no cover
log.error(f"object creation failed: {e}")
if self._sem:
self._sem.release()
_ = self._debug and self._log_debug(f"sem get R {self._sem._value}/{self._sem._initial_value}")
raise
obj = self._avail.pop()
self._using.add(obj)
self._nuses += 1
self._uses[obj].uses += 1
self._uses[obj].last_get = self._now()
_ = self._debug and log.debug(f"get {self._tracer(obj)}")
if self._getter:
try:
self._getter(obj)
except Exception as e:
log.error(f"exception in getter: {e}")
return obj
def ret(self, obj):
"""Return object to pool."""
_ = self._debug and log.debug(f"ret {self._tracer(obj)}")
if self._retter:
try:
self._retter(obj)
except Exception as e:
log.error(f"exception in retter: {e}")
with self._lock:
if obj not in self._using:
# multiple return, killed?
# NOTE cannot show object which could be in any state…
try:
log.warning(f"unexpected object returned: {self._tracer(obj)}")
except Exception as e:
log.error(f"exception in tracer on unexpected returned object: {e}")
return
if self._max_use and self._uses[obj].uses >= self._max_use:
self._nwornout += 1
self._out(obj)
self._todel.add(obj)
else:
self._using.remove(obj)
self._avail.add(obj)
self._uses[obj].last_ret = self._now()
if self._sem: # release token acquired in get()
self._sem.release()
_ = self._debug and self._log_debug(f"sem ret R {self._sem._value}/{self._sem._initial_value}")
self._empty()
self._fill()
@contextmanager
def obj(self, timeout=None):
"""Extract one object from the pool in a `with` scope."""
o = self.get(timeout)
try:
yield o
finally:
self.ret(o)