DEV-8: Added option do dump traceback

docs/configuration.rst

@@ -179,3 +179,10 @@ Logging
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 .. autopydantic_model:: LoggingConfig
+
+Debug
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. autopydantic_model:: DumpTracebackConfig
+
+.. autopydantic_model:: DebugConfig

src/HABApp/__version__.py

@@ -10,4 +10,4 @@
 # Development versions contain the DEV-COUNTER postfix:
 # - 24.01.0.DEV-1
 
-__version__ = '24.09.0.DEV-7'
+__version__ = '24.09.0.DEV-8'

src/HABApp/config/debug.py

@@ -0,0 +1,62 @@
+import logging
+from asyncio import sleep
+from faulthandler import dump_traceback
+from pathlib import Path
+
+import HABApp
+from HABApp.core.asyncio import create_task
+from HABApp.core.items.base_valueitem import datetime
+
+
+def _get_file_path(nr: int) -> Path:
+    assert nr >= 0  # noqa: S101
+    log_dir = HABApp.CONFIG.directories.logging
+    ctr = f'.{nr}' if nr else ''
+    return log_dir / f'HABApp_traceback{ctr:s}.log'
+
+
+def setup_debug() -> None:
+    debug = HABApp.CONFIG.habapp.debug
+    if (dump_traceback_cfg := debug.dump_traceback) is None:
+        return None
+
+    file = _get_file_path(0)
+    logging.getLogger('HABApp').info(f'Dumping traceback to {file}')
+
+    # rotate files
+    keep = 3
+    _get_file_path(keep).unlink(missing_ok=True)
+    for i in range(keep - 1, -1, -1):
+        src = _get_file_path(i)
+        if not src.is_file():
+            continue
+
+        dst = _get_file_path(i + 1)
+        dst.unlink(missing_ok=True)
+        src.rename(dst)
+
+    task = create_task(
+        dump_traceback_task(
+            file, int(dump_traceback_cfg.delay.total_seconds()), int(dump_traceback_cfg.interval.total_seconds())
+        ),
+        name='DumpTracebackTask'
+    )
+
+    HABApp.core.shutdown.register(task.cancel, msg='Stopping traceback task')
+
+
+async def dump_traceback_task(file: Path, delay: int, interval: int) -> None:
+
+    with file.open('a') as f:
+        f.write(f'Start: {datetime.now()}\n')
+        f.write(f'Delay: {delay:d}s Interval: {interval:d}s\n')
+
+    await sleep(delay)
+
+    while True:
+        with file.open('a') as f:
+            f.write(f'\n{"-" * 80}\n')
+            f.write(f'{datetime.now()}\n\n')
+            dump_traceback(f, all_threads=True)
+
+        await sleep(interval)

src/HABApp/config/loader.py

@@ -10,6 +10,7 @@
 from HABApp.config.config import CONFIG
 from HABApp.config.logging import HABAppQueueHandler
 
+from .debug import setup_debug
 from .errors import AbsolutePathExpected, InvalidConfigError
 from .logging import create_default_logfile, get_logging_dict, inject_log_buffer, rotate_files
 from .logging.buffered_logger import BufferedLogger
@@ -39,6 +40,8 @@ def load_config(config_folder: Path) -> None:
     if not loaded_logging:
         load_logging_cfg(logging_cfg_path)
 
+    setup_debug()
+
     # Watch folders, so we can reload the config on the fly
     filter = HABApp.core.files.watcher.FileEndingFilter('.yml')
     watcher = HABApp.core.files.watcher.AggregatingAsyncEventHandler(

src/HABApp/config/models/habapp.py

@@ -1,3 +1,5 @@
+from datetime import timedelta
+
 from easyconfig import BaseModel
 from pydantic import Field, conint
 
@@ -8,7 +10,7 @@ class ThreadPoolConfig(BaseModel):
     Use only if you have experience developing asyncio applications!
     If the thread pool is disabled using blocking calls in functions can and will break HABApp'''
 
-    threads: conint(ge=1, le=16) = 10
+    threads: conint(ge=1, le=32) = 10
     '''Amount of threads to use for the executor'''
 
 
@@ -20,8 +22,22 @@ class LoggingConfig(BaseModel):
     '''Wait time in seconds before the buffer gets flushed again when it was empty'''
 
 
+class DumpTracebackConfig(BaseModel):
+    delay: timedelta = Field('PT5M', gt=timedelta(seconds=0))
+    '''Initial delay before starting to dump the traceback'''
+
+    interval: timedelta = Field('PT1H', gt=timedelta(seconds=0))
+    '''Interval to dump the traceback'''
+
+
+class DebugConfig(BaseModel):
+    dump_traceback: DumpTracebackConfig | None = Field(None, alias='dump traceback')
+    '''Option to periodically dump the traceback of all currently running threads into a file'''
+
+
 class HABAppConfig(BaseModel):
     """HABApp internal configuration. Only change values if you know what you are doing!"""
 
     logging: LoggingConfig = Field(default_factory=LoggingConfig)
     thread_pool: ThreadPoolConfig = Field(default_factory=ThreadPoolConfig, alias='thread pool')
+    debug: DebugConfig = Field(default_factory=DebugConfig)

src/HABApp/core/const/const.py

@@ -3,6 +3,8 @@
 from enum import Enum
 from typing import Final
 
+from whenever import Instant
+
 
 class _MissingType(Enum):
     MISSING = object()
@@ -12,7 +14,8 @@ def __repr__(self) -> str:
 
 
 MISSING: Final = _MissingType.MISSING
-STARTUP: Final = time.monotonic()
+STARTUP_MONOTONIC: Final = time.monotonic()
+STARTUP_INSTANT: Final = Instant.now()
 
 # Python Versions for feature control
 PYTHON_311: Final = sys.version_info >= (3, 11)

src/HABApp/core/internals/event_bus/event_bus.py

@@ -42,7 +42,7 @@ def post_event(self, topic: str, event: Any) -> None:
             pass
 
         # Notify all listeners
-        if (listeners := self._listeners.get(topic, None)) is not None:
+        if (listeners := self._listeners.get(topic)) is not None:
             for listener in listeners:
                 listener.notify_listeners(event)
         return None
@@ -77,7 +77,7 @@ def remove_listener(self, listener: EventBusBaseListener) -> None:
             raise ValueError()
 
         with self._lock:
-            item_listeners = self._listeners.get(listener.topic, ())
+            item_listeners = self._listeners.get(topic, ())
 
             # print warning if we try to remove it twice
             if listener not in item_listeners:

src/HABApp/core/internals/wrapped_function/wrapped_thread.py

@@ -72,7 +72,7 @@ def __init__(self, parent: WrappedThreadFunction, func_obj: Callable[..., Any],
     def __repr__(self) -> str:
         return f'<{self.__class__.__name__} high: {self.usage_high:d}/{POOL_THREADS:d}>'
 
-    def run(self):
+    def run(self) -> Any:
         pool_lock: Final = POOL_LOCK
         pool_info: Final = POOL_INFO
         parent = self.parent

src/HABApp/core/shutdown.py

@@ -6,7 +6,7 @@
 import traceback
 from asyncio import iscoroutinefunction, sleep
 from dataclasses import dataclass
-from types import FunctionType, MethodType
+from types import BuiltinMethodType, FunctionType, MethodType
 from typing import TYPE_CHECKING
 
 from HABApp.core.asyncio import async_context, create_task
@@ -62,7 +62,7 @@ def register(func: Callable[[], Any], *, last: bool = False, msg: str = '') -> N
 
     if iscoroutinefunction(func):
         _REGISTERED += (ShutdownAwaitable(func=func, last=last, msg=msg), )
-    elif isinstance(func, (FunctionType, MethodType)):
+    elif isinstance(func, (FunctionType, MethodType, BuiltinMethodType)):
         _REGISTERED += (ShutdownFunction(func=func, last=last, msg=msg), )
     else:
         raise TypeError()
@@ -71,8 +71,11 @@ def register(func: Callable[[], Any], *, last: bool = False, msg: str = '') -> N
 async def _shutdown() -> None:
     global _REQUESTED
 
+    if _REQUESTED:
+        return None
     _REQUESTED = True
 
+
     async_context.set('Shutdown')
 
     log = logging.getLogger('HABApp.Shutdown')