feat: create TaskManager singleton class (#3775)

Co-authored-by: dadofsambonzuki <dadofsambonzuki@users.noreply.github.com>

lnbits/app.py

@@ -23,30 +23,35 @@ from lnbits.core.crud import (
     get_installed_extensions,
     update_installed_extension_state,
 )
+from lnbits.core.crud.audit import delete_expired_audit_entries
 from lnbits.core.crud.extensions import create_installed_extension
 from lnbits.core.helpers import migrate_extension_database
 from lnbits.core.models.notifications import NotificationType
 from lnbits.core.services.extensions import deactivate_extension, get_valid_extensions
-from lnbits.core.services.notifications import enqueue_admin_notification
+from lnbits.core.services.funding_source import (
-from lnbits.core.services.payments import check_pending_payments
+    check_balance_delta_changed,
+    check_server_balance_against_node,
+)
+from lnbits.core.services.notifications import (
+    dispatch_payment_notification,
+    enqueue_admin_notification,
+    process_next_notification,
+)
+from lnbits.core.services.payments import (
+    check_pending_payments,
+    fundingsource_invoice_producer,
+)
 from lnbits.core.tasks import (
     audit_queue,
     collect_exchange_rates_data,
-    purge_audit_data,
+    notify_server_status,
-    run_by_the_minute_tasks,
+    process_next_audit_entry,
-    wait_for_audit_data,
+    refresh_extension_cache,
-    wait_for_paid_invoices,
-    wait_notification_messages,
 )
 from lnbits.exceptions import register_exception_handlers
 from lnbits.helpers import version_parse
 from lnbits.llms_txt import create_llms_txt_route
 from lnbits.settings import settings
-from lnbits.tasks import (
-    cancel_all_tasks,
-    create_permanent_task,
-    register_invoice_listener,
-)
 from lnbits.utils.cache import cache
 from lnbits.utils.logger import (
     configure_logger,
@@ -69,7 +74,7 @@ from .middleware import (
     add_profiler_middleware,
     add_ratelimit_middleware,
 )
-from .tasks import internal_invoice_listener, invoice_listener, run_interval
+from .task_manager import task_manager
 
 
 async def startup(app: FastAPI):
@@ -133,7 +138,7 @@ async def shutdown():
     settings.lnbits_running = False
 
     # shutdown event
-    cancel_all_tasks()
+    task_manager.cancel_all_tasks()
 
     # wait a bit to allow them to finish, so that cleanup can run without problems
     await asyncio.sleep(0.1)
@@ -473,29 +478,42 @@ async def check_and_register_extensions(app: FastAPI) -> None:
 
 def register_async_tasks() -> None:
 
-    create_permanent_task(wait_for_audit_data)
+    task_manager.init()
-    create_permanent_task(wait_notification_messages)
 
-    create_permanent_task(
+    # listen to all incoming payments and dispatch payment notifications
-        run_interval(
+    # note: should be the first in task list for a bit quicker notifications
-            settings.lnbits_funding_source_pending_interval_seconds,
+    task_manager.register_invoice_listener(dispatch_payment_notification, "core")
+
+    # periodic tasks
+    task_manager.create_permanent_task(cache.invalidate_cache, interval=10)
+    task_manager.create_permanent_task(delete_expired_audit_entries, interval=60 * 60)
+    task_manager.create_permanent_task(
         check_pending_payments,
+        interval=settings.lnbits_funding_source_pending_interval_seconds,
     )
+    task_manager.create_permanent_task(
+        collect_exchange_rates_data,
+        interval=max(60, settings.lnbits_exchange_history_refresh_interval_seconds),
     )
-    create_permanent_task(invoice_listener)
+    task_manager.create_permanent_task(check_balance_delta_changed, interval=60)
-    create_permanent_task(internal_invoice_listener)
+    task_manager.create_permanent_task(
-    create_permanent_task(cache.invalidate_forever)
+        check_server_balance_against_node,
+        interval=60 * settings.lnbits_watchdog_interval_minutes,
+    )
+    task_manager.create_permanent_task(
+        notify_server_status,
+        interval=60 * 60 * settings.lnbits_notification_server_status_hours,
+    )
+    task_manager.create_permanent_task(refresh_extension_cache, interval=60)
 
-    # core invoice listener
+    # permanent tasks run in a loop, will be restarted if they fail
-    invoice_queue: asyncio.Queue = asyncio.Queue()
+    task_manager.create_permanent_task(fundingsource_invoice_producer)
-    register_invoice_listener(invoice_queue, "core")
+    task_manager.create_permanent_task(process_next_notification)
-    create_permanent_task(lambda: wait_for_paid_invoices(invoice_queue))
+    task_manager.create_permanent_task(process_next_audit_entry)
-
-    create_permanent_task(run_by_the_minute_tasks)
-    create_permanent_task(purge_audit_data)
-    create_permanent_task(collect_exchange_rates_data)
 
     # server logs for websocket
     if settings.lnbits_admin_ui:
         server_log_task = initialize_server_websocket_logger()
-        create_permanent_task(server_log_task)
+        task_manager.create_permanent_task(
+            server_log_task, name="server_websocket_logger"
+        )

lnbits/core/models/extensions.py

@@ -6,7 +6,6 @@ import json
 import os
 import shutil
 import zipfile
-from asyncio.tasks import create_task
 from pathlib import Path
 from typing import Any
 
@@ -21,6 +20,7 @@ from lnbits.helpers import (
     version_parse,
 )
 from lnbits.settings import settings
+from lnbits.task_manager import task_manager
 from lnbits.utils.cache import cache
 
 
@@ -642,7 +642,10 @@ class InstallableExtension(BaseModel):
 
         if cache_value.older_than(10 * 60) or post_refresh_cache:
             # refresh cache in background if older than 10 minutes or requested
-            create_task(cls._refresh_installable_extensions_cache())
+            task_manager.create_task(
+                cls._refresh_installable_extensions_cache(),
+                "refresh_installable_extensions_cache",
+            )
 
         extension_list = cache_value.value  # type: ignore
         return extension_list

lnbits/core/services/fiat_providers.py

@@ -20,6 +20,7 @@ from lnbits.fiat.base import (
     FiatPaymentSuccessStatus,
 )
 from lnbits.settings import settings
+from lnbits.task_manager import task_manager
 
 
 async def handle_fiat_payment_confirmation(
@@ -60,11 +61,7 @@ async def check_fiat_status(payment: Payment) -> FiatPaymentStatus:
         payment.status = PaymentState.SUCCESS.value
         await update_payment(payment)
         await handle_fiat_payment_confirmation(payment)
-
+        task_manager.internal_invoice_queue.put_nowait(payment)
-        # notify receivers asynchronously
-        from lnbits.tasks import internal_invoice_queue
-
-        await internal_invoice_queue.put(payment.checking_id)
 
     return fiat_status
 

lnbits/core/services/funding_source.py

@@ -66,6 +66,8 @@ async def check_server_balance_against_node():
 
 
 async def check_balance_delta_changed():
+    if settings.notification_balance_delta_threshold_sats <= 0:
+        return
     status = await get_balance_delta()
     if settings.latest_balance_delta_sats is None:
         settings.latest_balance_delta_sats = status.delta_sats

lnbits/core/services/notifications.py

@@ -237,6 +237,15 @@ async def send_email(
         return False
 
 
+async def dispatch_payment_notification(payment: Payment) -> None:
+    """
+    This worker dispatches the payment notifications.
+    """
+    wallet = await get_wallet(payment.wallet_id)
+    if wallet:
+        await send_payment_notification(wallet, payment)
+
+
 async def dispatch_webhook(payment: Payment):
     """
     Dispatches the webhook to the webhook url.

lnbits/core/services/payments.py

@@ -19,6 +19,7 @@ from lnbits.exceptions import InvoiceError, PaymentError, UnsupportedError
 from lnbits.fiat import get_fiat_provider
 from lnbits.helpers import check_callback_url
 from lnbits.settings import settings
+from lnbits.task_manager import task_manager
 from lnbits.utils.crypto import fake_privkey, random_secret_and_hash, verify_preimage
 from lnbits.utils.exchange_rates import fiat_amount_as_satoshis, satoshis_amount_as_fiat
 from lnbits.wallets import fake_wallet, get_funding_source
@@ -516,9 +517,7 @@ async def update_wallet_balance(
         )
         payment.status = PaymentState.SUCCESS
         await update_payment(payment, conn=conn)
-        from lnbits.tasks import internal_invoice_queue_put
+        task_manager.internal_invoice_queue.put_nowait(payment)
-
-        await internal_invoice_queue_put(payment.checking_id)
 
 
 async def check_wallet_limits(
@@ -789,10 +788,8 @@ async def _pay_internal_invoice(
     )  # notify the receiver
 
     # notify receiver asynchronously (extension listeners)
-    from lnbits.tasks import internal_invoice_queue
-
     logger.debug(f"enqueuing internal invoice {internal_payment.checking_id}")
-    await internal_invoice_queue.put(internal_payment.checking_id)
+    task_manager.internal_invoice_queue.put_nowait(internal_payment)
 
     return payment
 
@@ -829,16 +826,15 @@ async def _pay_external_invoice(
 
     fee_reserve_msat = fee_reserve(amount_msat, internal=False)
 
-    from lnbits.tasks import create_task
+    task = task_manager.create_task(
-
+        _fundingsource_pay_invoice(checking_id, payment.bolt11, fee_reserve_msat),
-    task = create_task(
+        f"fundingsource_pay_invoice_{checking_id}",
-        _fundingsource_pay_invoice(checking_id, payment.bolt11, fee_reserve_msat)
     )
 
     # make sure a hold invoice or deferred payment is not blocking the server
     wait_time = max(1, settings.lnbits_funding_source_pay_invoice_wait_seconds)
     try:
-        payment_response = await asyncio.wait_for(task, timeout=wait_time)
+        payment_response = await asyncio.wait_for(task.task, timeout=wait_time)
     except asyncio.TimeoutError:
         # return pending payment on timeout
         logger.debug(
@@ -1108,3 +1104,18 @@ async def update_invoice_from_paid_invoices_stream(checking_id: str) -> Payment
     payment = await update_payment(payment)
 
     return payment
+
+
+async def fundingsource_invoice_producer() -> None:
+    """
+    will collect all invoices that come directly from the backend wallet.
+
+    Called registered in the app startup sequence and run by taskmanager.
+    """
+    funding_source = get_funding_source()
+    async for checking_id in funding_source.paid_invoices_stream():
+        logger.info(f"got a payment notification {checking_id}")
+        payment = await update_invoice_from_paid_invoices_stream(checking_id)
+        if payment:
+            logger.success(f"fundingsource invoice {checking_id} settled")
+            task_manager.invoice_queue.put_nowait(payment)

lnbits/core/tasks.py

@@ -2,26 +2,16 @@ import asyncio
 
 from loguru import logger
 
-from lnbits.core.crud import (
+from lnbits.core.crud import create_audit_entry
-    create_audit_entry,
-    get_wallet,
-)
-from lnbits.core.crud.audit import delete_expired_audit_entries
 from lnbits.core.crud.payments import get_payments_status_count
 from lnbits.core.crud.users import get_accounts
 from lnbits.core.crud.wallets import get_wallets_count
 from lnbits.core.models.audit import AuditEntry
 from lnbits.core.models.extensions import InstallableExtension
 from lnbits.core.models.notifications import NotificationType
-from lnbits.core.services.funding_source import (
+from lnbits.core.services.funding_source import get_balance_delta
-    check_balance_delta_changed,
-    check_server_balance_against_node,
-    get_balance_delta,
-)
 from lnbits.core.services.notifications import (
     enqueue_admin_notification,
-    process_next_notification,
-    send_payment_notification,
 )
 from lnbits.db import Filters
 from lnbits.settings import settings
@@ -31,48 +21,24 @@ from lnbits.utils.exchange_rates import btc_price_from_aggregator, btc_rates
 audit_queue: asyncio.Queue[AuditEntry] = asyncio.Queue()
 
 
-async def run_by_the_minute_tasks() -> None:
+async def process_next_audit_entry() -> None:
-    minute_counter = 0
+    """
-    while settings.lnbits_running:
+    Waits for audit entries to be pushed to the queue.
-        status_minutes = settings.lnbits_notification_server_status_hours * 60
+    Then it inserts the entries into the DB.
-
+    """
-        if settings.notification_balance_delta_threshold_sats > 0:
+    data = await audit_queue.get()
-            try:
+    await create_audit_entry(data)
-                # runs by default every minute, the delta should not change that often
-                await check_balance_delta_changed()
-            except Exception as ex:
-                logger.error(ex)
-
-        if minute_counter % settings.lnbits_watchdog_interval_minutes == 0:
-            try:
-                await check_server_balance_against_node()
-            except Exception as ex:
-                logger.error(ex)
-
-        if minute_counter % status_minutes == 0:
-            try:
-                await _notify_server_status()
-            except Exception as ex:
-                logger.error(ex)
-
-        if minute_counter % 60 == 0:
-            try:
-                # initialize the list of all extensions
-                await InstallableExtension.get_installable_extensions(
-                    post_refresh_cache=True
-                )
-            except Exception as ex:
-                logger.error(ex)
-
-        minute_counter += 1
-        await asyncio.sleep(60)
 
 
-async def _notify_server_status() -> None:
+async def refresh_extension_cache() -> None:
+    # only refreshes every 10 minutes
+    await InstallableExtension.get_installable_extensions()
+
+
+async def notify_server_status() -> None:
     accounts = await get_accounts(filters=Filters(limit=0))
     wallets_count = await get_wallets_count()
     payments = await get_payments_status_count()
-
     status = await get_balance_delta()
     values = {
         "up_time": settings.lnbits_server_up_time,
@@ -89,67 +55,12 @@ async def _notify_server_status() -> None:
     enqueue_admin_notification(NotificationType.server_status, values)
 
 
-async def wait_for_paid_invoices(invoice_paid_queue: asyncio.Queue) -> None:
-    """
-    This worker dispatches events to all extensions and dispatches webhooks.
-    """
-    while settings.lnbits_running:
-        payment = await invoice_paid_queue.get()
-        logger.trace("received invoice paid event")
-        # payment notification
-        wallet = await get_wallet(payment.wallet_id)
-        if wallet:
-            await send_payment_notification(wallet, payment)
-
-
-async def wait_for_audit_data() -> None:
-    """
-    Waits for audit entries to be pushed to the queue.
-    Then it inserts the entries into the DB.
-    """
-    while settings.lnbits_running:
-        data = await audit_queue.get()
-        try:
-            await create_audit_entry(data)
-        except Exception as ex:
-            logger.warning(ex)
-            await asyncio.sleep(3)
-
-
-async def wait_notification_messages() -> None:
-
-    while settings.lnbits_running:
-        try:
-            await process_next_notification()
-        except Exception as ex:
-            logger.warning("Payment notification error", ex)
-            await asyncio.sleep(3)
-
-
-async def purge_audit_data() -> None:
-    """
-    Remove audit entries which have passed their retention period.
-    """
-    while settings.lnbits_running:
-        try:
-            await delete_expired_audit_entries()
-        except Exception as ex:
-            logger.warning(ex)
-
-        # clean every hour
-        await asyncio.sleep(60 * 60)
-
-
 async def collect_exchange_rates_data() -> None:
     """
     Collect exchange rates data. Used for monitoring only.
     """
-    while settings.lnbits_running:
     currency = settings.lnbits_default_accounting_currency or "USD"
     max_history_size = settings.lnbits_exchange_history_size
-        sleep_time = settings.lnbits_exchange_history_refresh_interval_seconds
-
-        if sleep_time > 0:
     try:
         if (
             settings.lnbits_price_aggregator_enabled
@@ -176,11 +87,6 @@ async def collect_exchange_rates_data() -> None:
                     lnbits_rate,
                     expiry=settings.lnbits_exchange_rate_cache_seconds,
                 )
-                    settings.append_exchange_rate_datapoint(
+            settings.append_exchange_rate_datapoint(dict(rates), max_history_size)
-                        dict(rates), max_history_size
-                    )
     except Exception as ex:
         logger.warning(ex)
-        else:
-            sleep_time = 60
-        await asyncio.sleep(sleep_time)

lnbits/core/views/admin_api.py

@@ -20,7 +20,7 @@ from lnbits.core.services.settings import dict_to_settings
 from lnbits.decorators import check_admin, check_super_user
 from lnbits.server import server_restart
 from lnbits.settings import AdminSettings, Settings, UpdateSettings, settings
-from lnbits.tasks import invoice_listeners
+from lnbits.task_manager import PublicTask, task_manager
 
 from .. import core_app_extra
 from ..crud import get_admin_settings, reset_core_settings, update_admin_settings
@@ -44,11 +44,10 @@ async def api_auditor():
     name="Monitor",
     description="show the current listeners and other monitoring data",
     dependencies=[Depends(check_admin)],
+    response_model=list[PublicTask],
 )
-async def api_monitor():
+async def api_monitor() -> list[PublicTask]:
-    return {
+    return task_manager.get_public_tasks()
-        "invoice_listeners": list(invoice_listeners.keys()),
-    }
 
 
 @admin_router.get(

lnbits/settings.py

@@ -1077,11 +1077,12 @@ class EnvSettings(LNbitsSettings):
     log_rotation: str = Field(default="100 MB")
     log_retention: str = Field(default="3 months")
     first_install_token: str | None = Field(default=None)
-
     cleanup_wallets_days: int = Field(default=90, ge=0)
     funding_source_max_retries: int = Field(default=4, ge=0)
     lnbits_max_users: int = Field(default=0, ge=0)
     lnbits_max_extensions: int = Field(default=0, ge=0)
+    task_heart_beat_verbose: bool = Field(default=False)
+    task_heart_beat_interval: int = Field(default=30)
 
     @property
     def has_default_extension_path(self) -> bool:

lnbits/task_manager.py

@@ -0,0 +1,200 @@
+import asyncio
+import traceback
+import uuid
+from collections.abc import Callable, Coroutine
+from datetime import datetime, timezone
+
+from loguru import logger
+from pydantic import BaseModel
+
+from lnbits.core.models import Payment
+from lnbits.settings import settings
+
+
+class PublicTask(BaseModel):
+    """Public model used to expose task information via the API."""
+
+    name: str
+    created_at: datetime
+
+
+class Task:
+    """Model used on the backend to keep track of background tasks."""
+
+    coro: Coroutine
+    name: str
+    created_at: datetime
+    task: asyncio.Task
+    invoice_queue: asyncio.Queue[Payment] | None = None
+
+    def __init__(
+        self,
+        coro: Coroutine,
+        name: str | None = None,
+        invoice_queue: asyncio.Queue | None = None,
+    ) -> None:
+        self.coro = coro
+        self.name = name or f"task_{uuid.uuid4()}"
+        self.created_at = datetime.now(timezone.utc)
+        self.task = asyncio.create_task(self.coro, name=self.name)
+        self.invoice_queue = invoice_queue
+
+
+class TaskManager:
+    """Singleton class to manage background tasks."""
+
+    tasks: list[Task] = []
+    invoice_queue: asyncio.Queue[Payment] = asyncio.Queue()
+    internal_invoice_queue: asyncio.Queue[Payment] = asyncio.Queue()
+
+    def init(self) -> None:
+        self.create_permanent_task(
+            func=self._heart_beat,
+            interval=settings.task_heart_beat_interval,
+        )
+        self.create_permanent_task(self._invoice_listener_consumer)
+        self.create_permanent_task(self._internal_invoice_listener_consumer)
+
+    def get_task(self, name: str) -> Task | None:
+        """Get a running task by name."""
+        for task in self.tasks:
+            if task.name == name:
+                return task
+        return None
+
+    def get_public_tasks(self) -> list[PublicTask]:
+        """Get a list of public tasks."""
+        return [PublicTask(name=t.name, created_at=t.created_at) for t in self.tasks]
+
+    def cancel_task(self, task: Task) -> None:
+        """Cancel a running task."""
+        self.tasks.remove(task)
+        try:
+            task.task.cancel()
+        except Exception as exc:
+            logger.warning(f"error while cancelling task `{task.name}`: {exc!s}")
+
+    def cancel_all_tasks(self) -> None:
+        """Cancel all running tasks."""
+        for task in list(self.tasks):
+            self.cancel_task(task)
+
+    def create_task(
+        self,
+        coro: Coroutine,
+        name: str | None = None,
+        invoice_queue: asyncio.Queue | None = None,
+    ) -> Task:
+        """Create a task. If a task with the same name exists, it will be cancelled."""
+        if name:
+            task = self.get_task(name)
+            if task:
+                self.cancel_task(task)
+        task = Task(coro=coro, name=name, invoice_queue=invoice_queue)
+        self.tasks.append(task)
+        return task
+
+    def create_permanent_task(
+        self,
+        func: Callable[[], Coroutine],
+        invoice_queue: asyncio.Queue | None = None,
+        name: str | None = None,
+        interval: int = 0,
+    ) -> Task:
+        """Create a task that runs forever and restarts on failure."""
+
+        async def wrapper():
+            while settings.lnbits_running:
+                await self._catch_everything_and_restart(func)
+                if interval > 0:
+                    await asyncio.sleep(interval)
+
+        return self.create_task(
+            coro=wrapper(), name=name or func.__name__, invoice_queue=invoice_queue
+        )
+
+    def register_invoice_listener(
+        self,
+        func: Callable[[Payment], Coroutine],
+        name: str | None = None,
+    ) -> Task:
+        """
+        A method intended for extensions to call when they want to be notified about
+        incoming payments. Will call provided Coroutine with the updated payment.
+        """
+        name = f"{name or uuid.uuid4()}_invoice_listener"
+        queue: asyncio.Queue[Payment] = asyncio.Queue()
+        return self.create_permanent_task(
+            self._invoice_listener_worker(func, queue),
+            name=name,
+            invoice_queue=queue,
+        )
+
+    async def _heart_beat(self) -> None:
+        """A heartbeat that removes done tasks logs the number of tasks."""
+        for task in self.tasks:
+            state = task.task._state if task.task else "NOT RUNNING"
+            if settings.task_heart_beat_verbose:
+                logger.debug(
+                    f"Task Manager: `{task.name}` state: `{state}` "
+                    f"created: {task.created_at.strftime('%Y-%m-%d %H:%M:%S')}`"
+                )
+            if task.task and task.task.done():
+                logger.debug(f"Task Manager: task `{task.name}` is done.")
+                self.cancel_task(task)
+        listeners_count = sum(1 for task in self.tasks if task.invoice_queue)
+        logger.debug(
+            f"Task Manager: {len(self.tasks) - listeners_count} tasks "
+            f"and {listeners_count} invoice listeners."
+        )
+
+    async def _catch_everything_and_restart(
+        self,
+        func: Callable[[], Coroutine],
+        restart_interval: int = 5,
+    ) -> None:
+        """Catches all exceptions from a function and restarts it after 5 seconds."""
+        while settings.lnbits_running:
+            try:
+                return await func()
+            except asyncio.CancelledError:
+                raise  # because we must pass this up
+            except Exception as exc:
+                if not settings.lnbits_running:
+                    return
+                logger.error(f"exception in background task `{func.__name__}`:", exc)
+                logger.error(traceback.format_exc())
+                logger.info(
+                    f"`{func.__name__}` restarts in {restart_interval} seconds."
+                )
+                await asyncio.sleep(restart_interval)
+
+    def _invoice_listener_worker(
+        self, func: Callable[[Payment], Coroutine], queue: asyncio.Queue[Payment]
+    ) -> Callable:
+        async def wrapper() -> None:
+            payment: Payment = await queue.get()
+            await func(payment)
+
+        return wrapper
+
+    def _invoice_dispatcher(self, payment: Payment) -> None:
+        """Dispatches a payment to all registered invoice listeners."""
+        for task in self.tasks:
+            if not task.invoice_queue:
+                continue
+            logger.debug(f"Enqueing payment to task {task.name}")
+            task.invoice_queue.put_nowait(payment)
+
+    async def _invoice_listener_consumer(self) -> None:
+        payment = await self.invoice_queue.get()
+        logger.info(f"got a payment notification {payment.checking_id}")
+        self._invoice_dispatcher(payment)
+
+    async def _internal_invoice_listener_consumer(self) -> None:
+        payment = await self.internal_invoice_queue.get()
+        logger.info(f"got an internal payment notification {payment.checking_id}")
+        self._invoice_dispatcher(payment)
+
+
+task_manager = TaskManager()

lnbits/tasks.py

@@ -1,146 +1,83 @@
 import asyncio
-import traceback
 import uuid
 from collections.abc import Callable, Coroutine
 
 from loguru import logger
 
 from lnbits.core.models import Payment
-from lnbits.core.services.payments import (
+from lnbits.core.services.payments import get_standalone_payment
-    get_standalone_payment,
-    update_invoice_from_paid_invoices_stream,
-)
 from lnbits.settings import settings
-from lnbits.wallets import get_funding_source
+from lnbits.task_manager import task_manager
-
-tasks: list[asyncio.Task] = []
-unique_tasks: dict[str, asyncio.Task] = {}
 
 
+# DEPRECATED: use task_manager.create_task instead.
 def create_task(coro: Coroutine) -> asyncio.Task:
-    task = asyncio.create_task(coro)
+    logger.debug("DEPRECATED: use task_manager.create_task instead.")
-    tasks.append(task)
+    return task_manager.create_task(coro).task
-    return task
 
 
+# DEPRECATED: use task_manager.create_task with `name` kwarg.
 def create_unique_task(name: str, coro: Coroutine) -> asyncio.Task:
-    if unique_tasks.get(name):
+    logger.debug("DEPRECATED: use task_manager.create_task instead.")
-        logger.warning(f"task `{name}` already exists, cancelling it")
+    return task_manager.create_task(coro, name=name).task
-        try:
-            unique_tasks[name].cancel()
-        except Exception as exc:
-            logger.warning(f"error while cancelling task `{name}`: {exc!s}")
-    task = asyncio.create_task(coro)
-    unique_tasks[name] = task
-    return task
 
 
+# DEPRECATED: use task_manager.create_permanent_task instead.
 def create_permanent_task(func: Callable[[], Coroutine]) -> asyncio.Task:
-    return create_task(catch_everything_and_restart(func))
+    logger.debug("DEPRECATED: use task_manager.create_permanent_task instead.")
+    return task_manager.create_permanent_task(func).task
 
 
+# DEPRECATED: use task_manager.create_permanent_task with `name` argument instead.
 def create_permanent_unique_task(
     name: str, coro: Callable[[], Coroutine]
 ) -> asyncio.Task:
     return create_unique_task(name, catch_everything_and_restart(coro, name))
 
 
-def cancel_all_tasks() -> None:
+# DEPRECATED don't use this, use task_manager.create_permanent_task instead.
-    for task in tasks:
-        try:
-            task.cancel()
-        except Exception as exc:
-            logger.warning(f"error while cancelling task: {exc!s}")
-    for name, task in unique_tasks.items():
-        try:
-            task.cancel()
-        except Exception as exc:
-            logger.warning(f"error while cancelling task `{name}`: {exc!s}")
-
-
 async def catch_everything_and_restart(
     func: Callable[[], Coroutine],
     name: str = "unnamed",
-) -> Coroutine:
+) -> None:
-    try:
+    _ = name
-        return await func()
+    return await task_manager._catch_everything_and_restart(func)
-    except asyncio.CancelledError:
-        raise  # because we must pass this up
-    except Exception as exc:
-        logger.error(f"exception in background task `{name}`:", exc)
-        logger.error(traceback.format_exc())
-        logger.error("will restart the task in 5 seconds.")
-        await asyncio.sleep(5)
-        return await catch_everything_and_restart(func, name)
 
 
-invoice_listeners: dict[str, asyncio.Queue] = {}
-
-
-# TODO: name should not be optional
-# some extensions still dont use a name, but they should
 def register_invoice_listener(send_chan: asyncio.Queue, name: str | None = None):
     """
-    A method intended for extensions (and core/tasks.py) to call when they want to be
+    DEPRECATED: use task_manager.register_invoice_listener instead,
-    notified about new invoice payments incoming. Will emit all incoming payments.
+    which also allows to pass a callback instead of a queue.
+    This method will still work but it is not recommended for new code.
     """
-    if not name:
+    logger.debug("DEPRECATED: use task_manager.register_invoice_listener instead.")
-        # fallback to a random name if extension didn't provide one
+    name = f"forward_{name or str(uuid.uuid4())[:8]}"
-        name = f"no_name_{str(uuid.uuid4())[:8]}"
 
-    if invoice_listeners.get(name):
+    # here we just forwarding the payments to the provided queue
-        logger.warning(f"invoice listener `{name}` already exists, replacing it")
+    async def forward_queue(payment: Payment):
+        send_chan.put_nowait(payment)
 
-    logger.trace(f"registering invoice listener `{name}`")
+    task_manager.register_invoice_listener(forward_queue, name=name)
-    invoice_listeners[name] = send_chan
-
-
-internal_invoice_queue: asyncio.Queue = asyncio.Queue(0)
 
 
 async def internal_invoice_queue_put(checking_id: str) -> None:
     """
+    DEPRECATED: use task_manager.internal_invoice_queue instead,
     A method to call when it wants to notify about an internal invoice payment.
     """
-    await internal_invoice_queue.put(checking_id)
-
-
-async def internal_invoice_listener() -> None:
-    """
-    internal_invoice_queue will be filled directly in core/services.py
-    after the payment was deemed to be settled internally.
-
-    Called by the app startup sequence.
-    """
-    while settings.lnbits_running:
-        checking_id = await internal_invoice_queue.get()
-        logger.info(f"got an internal payment notification {checking_id}")
     payment = await get_standalone_payment(checking_id, incoming=True)
-        if payment:
+    if not payment:
-            logger.success(f"internal invoice {checking_id} settled")
+        logger.warning(f"internal_invoice_queue_put: payment {checking_id} not found")
-            await invoice_callback_dispatcher(payment)
+        return
-
+    await task_manager.internal_invoice_queue.put(payment)
-
-async def invoice_listener() -> None:
-    """
-    invoice_listener will collect all invoices that come directly
-    from the backend wallet.
-
-    Called by the app startup sequence.
-    """
-    funding_source = get_funding_source()
-    async for checking_id in funding_source.paid_invoices_stream():
-        logger.info(f"got a payment notification {checking_id}")
-        payment = await update_invoice_from_paid_invoices_stream(checking_id)
-        if payment:
-            logger.success(f"fundingsource invoice {checking_id} settled")
-            await invoice_callback_dispatcher(payment)
 
 
+# DEPRECATED use task_manager.register_invoice_listener(coro, name="myext")
 def wait_for_paid_invoices(
     invoice_listener_name: str,
     func: Callable[[Payment], Coroutine],
 ) -> Callable[[], Coroutine]:
+    logger.debug("DEPRECATED: use task_manager.register_invoice_listener instead.")
 
     async def wrapper() -> None:
         invoice_queue: asyncio.Queue = asyncio.Queue()
@@ -150,27 +87,3 @@ def wait_for_paid_invoices(
             await func(payment)
 
     return wrapper
-
-
-def run_interval(
-    interval_seconds: int,
-    func: Callable[[], Coroutine],
-) -> Callable[[], Coroutine]:
-    """Run a function at a specified interval in seconds, while the server is running"""
-
-    async def wrapper() -> None:
-        while settings.lnbits_running:
-            try:
-                await func()
-            except Exception as e:
-                logger.error(f"Error occurred in interval task: {e}")
-                logger.warning(traceback.format_exc())
-            await asyncio.sleep(interval_seconds)
-
-    return wrapper
-
-
-async def invoice_callback_dispatcher(payment: Payment):
-    for name, send_chan in invoice_listeners.items():
-        logger.trace(f"invoice listeners: sending to `{name}`")
-        await send_chan.put(payment)

lnbits/utils/cache.py

@@ -1,13 +1,8 @@
 from __future__ import annotations
 
-import asyncio
 from time import time
 from typing import Any, NamedTuple
 
-from loguru import logger
-
-from lnbits.settings import settings
-
 
 class Cached(NamedTuple):
     value: Any
@@ -22,8 +17,7 @@ class Cache:
     Small caching utility providing simple get/set interface (very much like redis)
     """
 
-    def __init__(self, interval: float = 10) -> None:
+    def __init__(self) -> None:
-        self.interval = interval
         self._values: dict[Any, Cached] = {}
 
     def value(self, key: str) -> Cached | None:
@@ -59,16 +53,11 @@ class Cache:
             self.set(key, value, expiry=expiry)
             return value
 
-    async def invalidate_forever(self):
+    async def invalidate_cache(self):
-        while settings.lnbits_running:
-            try:
-                await asyncio.sleep(self.interval)
         ts = time()
         expired = [k for k, v in self._values.items() if v.expiry < ts]
         for k in expired:
             self._values.pop(k)
-            except Exception:
-                logger.error("Error invalidating cache")
 
 
 cache = Cache()

lnbits/utils/logger.py

@@ -41,19 +41,16 @@ def log_server_info():
 
 def initialize_server_websocket_logger() -> Callable:
     super_user_hash = sha256(settings.super_user.encode("utf-8")).hexdigest()
-
     serverlog_queue: asyncio.Queue = asyncio.Queue()
-
-    async def update_websocket_serverlog():
-        while settings.lnbits_running:
-            msg = await serverlog_queue.get()
-            await websocket_updater(super_user_hash, msg)
-
     logger.add(
         lambda msg: serverlog_queue.put_nowait(msg),
         format=Formatter().format,
     )
 
+    async def update_websocket_serverlog():
+        msg = await serverlog_queue.get()
+        await websocket_updater(super_user_hash, msg)
+
     return update_websocket_serverlog
 
 

tests/api/test_admin_api.py

@@ -82,7 +82,8 @@ async def test_admin_audit_monitor_and_test_email(
         headers={"Authorization": f"Bearer {superuser_token}"},
     )
     assert monitor.status_code == 200
-    assert "invoice_listeners" in monitor.json()
+    task_names = [t["name"] for t in monitor.json()]
+    assert any("invoice_listener" in name for name in task_names)
 
     test_email = await client.get(
         "/admin/api/v1/testemail",

tests/regtest/test_real_invoice.py

@@ -13,10 +13,13 @@ from lnbits.core.services import (
     fee_reserve_total,
     get_balance_delta,
 )
-from lnbits.core.services.payments import pay_invoice, update_wallet_balance
+from lnbits.core.services.payments import (
+    pay_invoice,
+    update_wallet_balance,
+)
 from lnbits.core.services.users import create_user_account
 from lnbits.exceptions import PaymentError
-from lnbits.tasks import create_task, wait_for_paid_invoices
+from lnbits.task_manager import task_manager
 from lnbits.wallets import get_funding_source
 
 from ..helpers import is_fake, is_regtest
@@ -160,12 +163,11 @@ async def test_create_real_invoice(
     assert not payment_status["paid"]
 
     on_paid_mock = mocker.AsyncMock()
-    create_task(wait_for_paid_invoices("test_create_invoice", on_paid_mock)())
+    task_manager.register_invoice_listener(on_paid_mock, "test_create_invoice")
 
     pay_real_invoice(invoice["bolt11"])
 
     await asyncio.sleep(1)
-
     assert on_paid_mock.call_count == 1
     payment = on_paid_mock.call_args_list[0][0][0]
 
@@ -393,12 +395,11 @@ async def test_receive_real_invoice_set_pending_and_check_state(
     assert not payment_status["paid"]
 
     on_paid_mock = mocker.AsyncMock()
-    create_task(wait_for_paid_invoices("test_create_invoice", on_paid_mock)())
+    task_manager.register_invoice_listener(on_paid_mock, "test_create_invoice")
 
     pay_real_invoice(invoice["bolt11"])
 
     await asyncio.sleep(1)
-
     assert on_paid_mock.call_count == 1
     payment = on_paid_mock.call_args_list[0][0][0]
 
@@ -412,6 +413,8 @@ async def test_receive_real_invoice_set_pending_and_check_state(
     payment_status = response.json()
     assert payment_status["paid"]
 
+    assert payment
+
     # set the incoming invoice to pending
     payment.status = PaymentState.PENDING
     await update_payment(payment)

tests/unit/test_cache.py

@@ -5,6 +5,7 @@ import pytest
 from pytest_mock.plugin import MockerFixture
 
 from lnbits.settings import Settings
+from lnbits.task_manager import task_manager
 from lnbits.utils.cache import Cache, Cached
 
 key = "foo"
@@ -13,11 +14,10 @@ value = "bar"
 
 @pytest.fixture
 async def cache():
-    cache = Cache(interval=0.1)
+    cache = Cache()
-
+    task = task_manager.create_permanent_task(cache.invalidate_cache, interval=1)
-    task = asyncio.create_task(cache.invalidate_forever())
     yield cache
-    task.cancel()
+    task_manager.cancel_task(task)
 
 
 @pytest.mark.anyio
@@ -31,13 +31,13 @@ async def test_cache_get_set(cache):
 @pytest.mark.anyio
 async def test_cache_expiry(cache):
     # gets expired by `get` call
-    cache.set(key, value, expiry=0.01)
+    cache.set(key, value, expiry=1)
-    await asyncio.sleep(0.02)
+    await asyncio.sleep(2)
     assert not cache.get(key)
 
     # gets expired by invalidation task
-    cache.set(key, value, expiry=0.1)
+    cache.set(key, value, expiry=1)
-    await asyncio.sleep(0.2)
+    await asyncio.sleep(2)
     assert key not in cache._values
     assert not cache.get(key)
 
@@ -94,23 +94,33 @@ async def test_cache_pop_expired_returns_default(cache):
 async def test_invalidate_forever_logs_and_recovers_from_errors(
     settings: Settings, mocker: MockerFixture
 ):
-    test_cache = Cache(interval=0)
+    test_cache = Cache()
-    logger_error = mocker.patch("lnbits.utils.cache.logger.error")
     original_running = settings.lnbits_running
     calls = 0
 
-    async def fake_sleep(_interval):
+    original_invalidate = test_cache.invalidate_cache
+
+    async def fake_invalidate():
         nonlocal calls
         calls += 1
         if calls == 1:
             raise RuntimeError("boom")
         settings.lnbits_running = False
+        await original_invalidate()
 
+    mocker.patch.object(test_cache, "invalidate_cache", side_effect=fake_invalidate)
+    mocker.patch("lnbits.task_manager.asyncio.sleep")
+    logger_error = mocker.patch("lnbits.task_manager.logger.error")
+
+    bg_task = None
     try:
         settings.lnbits_running = True
-        mocker.patch("lnbits.utils.cache.asyncio.sleep", side_effect=fake_sleep)
+        bg_task = task_manager.create_permanent_task(test_cache.invalidate_cache)
-        await test_cache.invalidate_forever()
+        await bg_task.task
     finally:
         settings.lnbits_running = original_running
+        if bg_task:
+            task_manager.cancel_task(bg_task)
 
-    logger_error.assert_called_once_with("Error invalidating cache")
+    assert logger_error.called
+    assert calls == 2

tests/unit/test_fiat_providers.py

@@ -1717,7 +1717,9 @@ async def test_check_fiat_status_handles_internal_states(mocker: MockerFixture):
         "lnbits.core.services.fiat_providers.get_fiat_provider",
         AsyncMock(return_value=provider),
     )
-    queue_put = mocker.patch("lnbits.tasks.internal_invoice_queue.put", AsyncMock())
+    queue_put = mocker.patch(
+        "lnbits.task_manager.task_manager.internal_invoice_queue.put_nowait"
+    )
 
     success_status = await check_fiat_status(
         Payment(
@@ -1734,7 +1736,8 @@ async def test_check_fiat_status_handles_internal_states(mocker: MockerFixture):
     )
 
     assert success_status.success is True
-    queue_put.assert_awaited_once_with("fiat_pending")
+    queue_put.assert_called_once()
+    assert queue_put.call_args[0][0].checking_id == "fiat_pending"
 
     await check_fiat_status(
         Payment(
@@ -1749,7 +1752,7 @@ async def test_check_fiat_status_handles_internal_states(mocker: MockerFixture):
             extra={"fiat_checking_id": "stripe_checking_id"},
         )
     )
-    assert queue_put.await_count == 1
+    assert queue_put.call_count == 1
 
 
 @pytest.mark.anyio
@@ -1786,7 +1789,9 @@ async def test_check_fiat_status_persists_successful_payment(
         "lnbits.fiat.StripeWallet.get_invoice_status",
         AsyncMock(return_value=FiatPaymentStatus(paid=True)),
     )
-    queue_put = mocker.patch("lnbits.tasks.internal_invoice_queue.put", AsyncMock())
+    queue_put = mocker.patch(
+        "lnbits.task_manager.task_manager.internal_invoice_queue.put_nowait"
+    )
 
     status = await check_fiat_status(payment)
 
@@ -1794,7 +1799,7 @@ async def test_check_fiat_status_persists_successful_payment(
     assert payment.status == PaymentState.SUCCESS
     updated_payment = await get_payment(payment.checking_id)
     assert updated_payment.status == PaymentState.SUCCESS
-    queue_put.assert_awaited_once_with(payment.checking_id)
+    queue_put.assert_called_once_with(payment)
 
 
 @pytest.mark.anyio

tests/unit/test_pay_invoice.py

@@ -12,15 +12,12 @@ from lnbits.core.crud import create_wallet, get_standalone_payment, get_wallet
 from lnbits.core.crud.payments import get_payment, get_payments_paginated
 from lnbits.core.models import PaymentState, Wallet
 from lnbits.core.services import create_invoice, create_user_account, pay_invoice
-from lnbits.core.services.payments import update_wallet_balance
+from lnbits.core.services.payments import (
+    update_wallet_balance,
+)
 from lnbits.exceptions import InvoiceError, PaymentError
 from lnbits.settings import Settings
-from lnbits.tasks import (
+from lnbits.task_manager import task_manager
-    create_task,
-    internal_invoice_listener,
-    internal_invoice_queue,
-    wait_for_paid_invoices,
-)
 from lnbits.wallets.base import PaymentResponse
 from lnbits.wallets.fake import FakeWallet
 
@@ -237,24 +234,30 @@ async def test_notification_for_internal_payment(
     test_name = "test_notification_for_internal_payment"
 
     # Drain stale items left by session-scoped fixtures (e.g. update_wallet_balance)
-    while not internal_invoice_queue.empty():
+    while not task_manager.internal_invoice_queue.empty():
         try:
-            internal_invoice_queue.get_nowait()
+            task_manager.internal_invoice_queue.get_nowait()
         except asyncio.QueueEmpty:
             break
 
     on_paid_mock = mocker.AsyncMock()
-    create_task(internal_invoice_listener())
+    # create_task(internal_invoice_listener())
-    create_task(wait_for_paid_invoices(test_name, on_paid_mock)())
+
+    task_manager.register_invoice_listener(on_paid_mock, test_name)
+
     payment = await create_invoice(
         wallet_id=to_wallet.id,
         amount=123,
         memo=test_name,
         webhook="http://test.404.lnbits.com",
     )
-    await pay_invoice(
+    paid_payment = await pay_invoice(
         wallet_id=to_wallet.id, payment_request=payment.bolt11, extra={"tag": "lnurlp"}
     )
+    assert paid_payment.status == PaymentState.SUCCESS.value
+    assert paid_payment.bolt11 == payment.bolt11
+    assert paid_payment.amount == -123_000
+
     await asyncio.sleep(1)
 
     assert on_paid_mock.call_count == 1
@@ -264,6 +267,8 @@ async def test_notification_for_internal_payment(
     assert _payment.status == PaymentState.SUCCESS.value
     assert _payment.bolt11 == payment.bolt11
     assert _payment.amount == 123_000
+    assert _payment.checking_id == payment.checking_id
+
     updated_payment = await get_payment(_payment.checking_id)
     assert (
         updated_payment.webhook_status is not None

tests/unit/test_services_payments.py

@@ -197,8 +197,7 @@ async def test_update_wallet_balance_validates_credit_and_debit(
 
         settings.lnbits_wallet_limit_max_balance = 0
         queue_mock = mocker.patch(
-            "lnbits.tasks.internal_invoice_queue_put",
+            "lnbits.task_manager.task_manager.internal_invoice_queue.put_nowait",
-            mocker.AsyncMock(),
         )
 
         await update_wallet_balance(wallet, 5)
@@ -212,7 +211,8 @@ async def test_update_wallet_balance_validates_credit_and_debit(
     ]
     assert credit_payments
     assert credit_payments[0].status == PaymentState.SUCCESS
-    queue_mock.assert_awaited_once_with(credit_payments[0].checking_id)
+    queue_mock.assert_called_once()
+    assert queue_mock.call_args[0][0].checking_id == credit_payments[0].checking_id
 
 
 @pytest.mark.anyio