diff --git a/lnbits/app.py b/lnbits/app.py index 46caa188c..c4c2d5651 100644 --- a/lnbits/app.py +++ b/lnbits/app.py @@ -23,30 +23,35 @@ from lnbits.core.crud import ( get_installed_extensions, update_installed_extension_state, ) +from lnbits.core.crud.audit import delete_expired_audit_entries from lnbits.core.crud.extensions import create_installed_extension from lnbits.core.helpers import migrate_extension_database from lnbits.core.models.notifications import NotificationType from lnbits.core.services.extensions import deactivate_extension, get_valid_extensions -from lnbits.core.services.notifications import enqueue_admin_notification -from lnbits.core.services.payments import check_pending_payments +from lnbits.core.services.funding_source import ( + check_balance_delta_changed, + check_server_balance_against_node, +) +from lnbits.core.services.notifications import ( + dispatch_payment_notification, + enqueue_admin_notification, + process_next_notification, +) +from lnbits.core.services.payments import ( + check_pending_payments, + fundingsource_invoice_producer, +) from lnbits.core.tasks import ( audit_queue, collect_exchange_rates_data, - purge_audit_data, - run_by_the_minute_tasks, - wait_for_audit_data, - wait_for_paid_invoices, - wait_notification_messages, + notify_server_status, + process_next_audit_entry, + refresh_extension_cache, ) from lnbits.exceptions import register_exception_handlers from lnbits.helpers import version_parse from lnbits.llms_txt import create_llms_txt_route from lnbits.settings import settings -from lnbits.tasks import ( - cancel_all_tasks, - create_permanent_task, - register_invoice_listener, -) from lnbits.utils.cache import cache from lnbits.utils.logger import ( configure_logger, @@ -69,7 +74,7 @@ from .middleware import ( add_profiler_middleware, add_ratelimit_middleware, ) -from .tasks import internal_invoice_listener, invoice_listener, run_interval +from .task_manager import task_manager async def startup(app: FastAPI): @@ -133,7 +138,7 @@ async def shutdown(): settings.lnbits_running = False # shutdown event - cancel_all_tasks() + task_manager.cancel_all_tasks() # wait a bit to allow them to finish, so that cleanup can run without problems await asyncio.sleep(0.1) @@ -473,29 +478,42 @@ async def check_and_register_extensions(app: FastAPI) -> None: def register_async_tasks() -> None: - create_permanent_task(wait_for_audit_data) - create_permanent_task(wait_notification_messages) + task_manager.init() - create_permanent_task( - run_interval( - settings.lnbits_funding_source_pending_interval_seconds, - check_pending_payments, - ) + # listen to all incoming payments and dispatch payment notifications + # note: should be the first in task list for a bit quicker notifications + task_manager.register_invoice_listener(dispatch_payment_notification, "core") + + # periodic tasks + task_manager.create_permanent_task(cache.invalidate_cache, interval=10) + task_manager.create_permanent_task(delete_expired_audit_entries, interval=60 * 60) + task_manager.create_permanent_task( + check_pending_payments, + interval=settings.lnbits_funding_source_pending_interval_seconds, ) - create_permanent_task(invoice_listener) - create_permanent_task(internal_invoice_listener) - create_permanent_task(cache.invalidate_forever) + task_manager.create_permanent_task( + collect_exchange_rates_data, + interval=max(60, settings.lnbits_exchange_history_refresh_interval_seconds), + ) + task_manager.create_permanent_task(check_balance_delta_changed, interval=60) + task_manager.create_permanent_task( + check_server_balance_against_node, + interval=60 * settings.lnbits_watchdog_interval_minutes, + ) + task_manager.create_permanent_task( + notify_server_status, + interval=60 * 60 * settings.lnbits_notification_server_status_hours, + ) + task_manager.create_permanent_task(refresh_extension_cache, interval=60) - # core invoice listener - invoice_queue: asyncio.Queue = asyncio.Queue() - register_invoice_listener(invoice_queue, "core") - create_permanent_task(lambda: wait_for_paid_invoices(invoice_queue)) - - create_permanent_task(run_by_the_minute_tasks) - create_permanent_task(purge_audit_data) - create_permanent_task(collect_exchange_rates_data) + # permanent tasks run in a loop, will be restarted if they fail + task_manager.create_permanent_task(fundingsource_invoice_producer) + task_manager.create_permanent_task(process_next_notification) + task_manager.create_permanent_task(process_next_audit_entry) # server logs for websocket if settings.lnbits_admin_ui: server_log_task = initialize_server_websocket_logger() - create_permanent_task(server_log_task) + task_manager.create_permanent_task( + server_log_task, name="server_websocket_logger" + ) diff --git a/lnbits/core/models/extensions.py b/lnbits/core/models/extensions.py index 5872ecb67..79e93e6fd 100644 --- a/lnbits/core/models/extensions.py +++ b/lnbits/core/models/extensions.py @@ -6,7 +6,6 @@ import json import os import shutil import zipfile -from asyncio.tasks import create_task from pathlib import Path from typing import Any @@ -21,6 +20,7 @@ from lnbits.helpers import ( version_parse, ) from lnbits.settings import settings +from lnbits.task_manager import task_manager from lnbits.utils.cache import cache @@ -642,7 +642,10 @@ class InstallableExtension(BaseModel): if cache_value.older_than(10 * 60) or post_refresh_cache: # refresh cache in background if older than 10 minutes or requested - create_task(cls._refresh_installable_extensions_cache()) + task_manager.create_task( + cls._refresh_installable_extensions_cache(), + "refresh_installable_extensions_cache", + ) extension_list = cache_value.value # type: ignore return extension_list diff --git a/lnbits/core/services/fiat_providers.py b/lnbits/core/services/fiat_providers.py index c298f78cf..6f3c6fdcc 100644 --- a/lnbits/core/services/fiat_providers.py +++ b/lnbits/core/services/fiat_providers.py @@ -20,6 +20,7 @@ from lnbits.fiat.base import ( FiatPaymentSuccessStatus, ) from lnbits.settings import settings +from lnbits.task_manager import task_manager async def handle_fiat_payment_confirmation( @@ -60,11 +61,7 @@ async def check_fiat_status(payment: Payment) -> FiatPaymentStatus: payment.status = PaymentState.SUCCESS.value await update_payment(payment) await handle_fiat_payment_confirmation(payment) - - # notify receivers asynchronously - from lnbits.tasks import internal_invoice_queue - - await internal_invoice_queue.put(payment.checking_id) + task_manager.internal_invoice_queue.put_nowait(payment) return fiat_status diff --git a/lnbits/core/services/funding_source.py b/lnbits/core/services/funding_source.py index 02caab058..ce9437dc1 100644 --- a/lnbits/core/services/funding_source.py +++ b/lnbits/core/services/funding_source.py @@ -66,6 +66,8 @@ async def check_server_balance_against_node(): async def check_balance_delta_changed(): + if settings.notification_balance_delta_threshold_sats <= 0: + return status = await get_balance_delta() if settings.latest_balance_delta_sats is None: settings.latest_balance_delta_sats = status.delta_sats diff --git a/lnbits/core/services/notifications.py b/lnbits/core/services/notifications.py index ddecfb11c..bed83e842 100644 --- a/lnbits/core/services/notifications.py +++ b/lnbits/core/services/notifications.py @@ -237,6 +237,15 @@ async def send_email( return False +async def dispatch_payment_notification(payment: Payment) -> None: + """ + This worker dispatches the payment notifications. + """ + wallet = await get_wallet(payment.wallet_id) + if wallet: + await send_payment_notification(wallet, payment) + + async def dispatch_webhook(payment: Payment): """ Dispatches the webhook to the webhook url. diff --git a/lnbits/core/services/payments.py b/lnbits/core/services/payments.py index 7f5e65846..fb3cd24f2 100644 --- a/lnbits/core/services/payments.py +++ b/lnbits/core/services/payments.py @@ -19,6 +19,7 @@ from lnbits.exceptions import InvoiceError, PaymentError, UnsupportedError from lnbits.fiat import get_fiat_provider from lnbits.helpers import check_callback_url from lnbits.settings import settings +from lnbits.task_manager import task_manager from lnbits.utils.crypto import fake_privkey, random_secret_and_hash, verify_preimage from lnbits.utils.exchange_rates import fiat_amount_as_satoshis, satoshis_amount_as_fiat from lnbits.wallets import fake_wallet, get_funding_source @@ -516,9 +517,7 @@ async def update_wallet_balance( ) payment.status = PaymentState.SUCCESS await update_payment(payment, conn=conn) - from lnbits.tasks import internal_invoice_queue_put - - await internal_invoice_queue_put(payment.checking_id) + task_manager.internal_invoice_queue.put_nowait(payment) async def check_wallet_limits( @@ -789,10 +788,8 @@ async def _pay_internal_invoice( ) # notify the receiver # notify receiver asynchronously (extension listeners) - from lnbits.tasks import internal_invoice_queue - logger.debug(f"enqueuing internal invoice {internal_payment.checking_id}") - await internal_invoice_queue.put(internal_payment.checking_id) + task_manager.internal_invoice_queue.put_nowait(internal_payment) return payment @@ -829,16 +826,15 @@ async def _pay_external_invoice( fee_reserve_msat = fee_reserve(amount_msat, internal=False) - from lnbits.tasks import create_task - - task = create_task( - _fundingsource_pay_invoice(checking_id, payment.bolt11, fee_reserve_msat) + task = task_manager.create_task( + _fundingsource_pay_invoice(checking_id, payment.bolt11, fee_reserve_msat), + f"fundingsource_pay_invoice_{checking_id}", ) # make sure a hold invoice or deferred payment is not blocking the server wait_time = max(1, settings.lnbits_funding_source_pay_invoice_wait_seconds) try: - payment_response = await asyncio.wait_for(task, timeout=wait_time) + payment_response = await asyncio.wait_for(task.task, timeout=wait_time) except asyncio.TimeoutError: # return pending payment on timeout logger.debug( @@ -1108,3 +1104,18 @@ async def update_invoice_from_paid_invoices_stream(checking_id: str) -> Payment payment = await update_payment(payment) return payment + + +async def fundingsource_invoice_producer() -> None: + """ + will collect all invoices that come directly from the backend wallet. + + Called registered in the app startup sequence and run by taskmanager. + """ + funding_source = get_funding_source() + async for checking_id in funding_source.paid_invoices_stream(): + logger.info(f"got a payment notification {checking_id}") + payment = await update_invoice_from_paid_invoices_stream(checking_id) + if payment: + logger.success(f"fundingsource invoice {checking_id} settled") + task_manager.invoice_queue.put_nowait(payment) diff --git a/lnbits/core/tasks.py b/lnbits/core/tasks.py index c5f41bc7f..e11e205bf 100644 --- a/lnbits/core/tasks.py +++ b/lnbits/core/tasks.py @@ -2,26 +2,16 @@ import asyncio from loguru import logger -from lnbits.core.crud import ( - create_audit_entry, - get_wallet, -) -from lnbits.core.crud.audit import delete_expired_audit_entries +from lnbits.core.crud import create_audit_entry from lnbits.core.crud.payments import get_payments_status_count from lnbits.core.crud.users import get_accounts from lnbits.core.crud.wallets import get_wallets_count from lnbits.core.models.audit import AuditEntry from lnbits.core.models.extensions import InstallableExtension from lnbits.core.models.notifications import NotificationType -from lnbits.core.services.funding_source import ( - check_balance_delta_changed, - check_server_balance_against_node, - get_balance_delta, -) +from lnbits.core.services.funding_source import get_balance_delta from lnbits.core.services.notifications import ( enqueue_admin_notification, - process_next_notification, - send_payment_notification, ) from lnbits.db import Filters from lnbits.settings import settings @@ -31,48 +21,24 @@ from lnbits.utils.exchange_rates import btc_price_from_aggregator, btc_rates audit_queue: asyncio.Queue[AuditEntry] = asyncio.Queue() -async def run_by_the_minute_tasks() -> None: - minute_counter = 0 - while settings.lnbits_running: - status_minutes = settings.lnbits_notification_server_status_hours * 60 - - if settings.notification_balance_delta_threshold_sats > 0: - try: - # runs by default every minute, the delta should not change that often - await check_balance_delta_changed() - except Exception as ex: - logger.error(ex) - - if minute_counter % settings.lnbits_watchdog_interval_minutes == 0: - try: - await check_server_balance_against_node() - except Exception as ex: - logger.error(ex) - - if minute_counter % status_minutes == 0: - try: - await _notify_server_status() - except Exception as ex: - logger.error(ex) - - if minute_counter % 60 == 0: - try: - # initialize the list of all extensions - await InstallableExtension.get_installable_extensions( - post_refresh_cache=True - ) - except Exception as ex: - logger.error(ex) - - minute_counter += 1 - await asyncio.sleep(60) +async def process_next_audit_entry() -> None: + """ + Waits for audit entries to be pushed to the queue. + Then it inserts the entries into the DB. + """ + data = await audit_queue.get() + await create_audit_entry(data) -async def _notify_server_status() -> None: +async def refresh_extension_cache() -> None: + # only refreshes every 10 minutes + await InstallableExtension.get_installable_extensions() + + +async def notify_server_status() -> None: accounts = await get_accounts(filters=Filters(limit=0)) wallets_count = await get_wallets_count() payments = await get_payments_status_count() - status = await get_balance_delta() values = { "up_time": settings.lnbits_server_up_time, @@ -89,98 +55,38 @@ async def _notify_server_status() -> None: enqueue_admin_notification(NotificationType.server_status, values) -async def wait_for_paid_invoices(invoice_paid_queue: asyncio.Queue) -> None: - """ - This worker dispatches events to all extensions and dispatches webhooks. - """ - while settings.lnbits_running: - payment = await invoice_paid_queue.get() - logger.trace("received invoice paid event") - # payment notification - wallet = await get_wallet(payment.wallet_id) - if wallet: - await send_payment_notification(wallet, payment) - - -async def wait_for_audit_data() -> None: - """ - Waits for audit entries to be pushed to the queue. - Then it inserts the entries into the DB. - """ - while settings.lnbits_running: - data = await audit_queue.get() - try: - await create_audit_entry(data) - except Exception as ex: - logger.warning(ex) - await asyncio.sleep(3) - - -async def wait_notification_messages() -> None: - - while settings.lnbits_running: - try: - await process_next_notification() - except Exception as ex: - logger.warning("Payment notification error", ex) - await asyncio.sleep(3) - - -async def purge_audit_data() -> None: - """ - Remove audit entries which have passed their retention period. - """ - while settings.lnbits_running: - try: - await delete_expired_audit_entries() - except Exception as ex: - logger.warning(ex) - - # clean every hour - await asyncio.sleep(60 * 60) - - async def collect_exchange_rates_data() -> None: """ Collect exchange rates data. Used for monitoring only. """ - while settings.lnbits_running: - currency = settings.lnbits_default_accounting_currency or "USD" - max_history_size = settings.lnbits_exchange_history_size - sleep_time = settings.lnbits_exchange_history_refresh_interval_seconds - - if sleep_time > 0: - try: - if ( - settings.lnbits_price_aggregator_enabled - and settings.lnbits_price_aggregator_url - ): - price = await btc_price_from_aggregator(currency) - if price: - cache.set( - f"btc-price-{currency}", - price, - expiry=settings.lnbits_exchange_rate_cache_seconds, - ) - settings.append_exchange_rate_datapoint( - {"Aggregator": price}, max_history_size - ) - else: - rates = await btc_rates(currency) - if rates: - rates_values = [r[1] for r in rates] - lnbits_rate = sum(rates_values) / len(rates_values) - rates.append(("LNbits", lnbits_rate)) - cache.set( - f"btc-price-{currency}", - lnbits_rate, - expiry=settings.lnbits_exchange_rate_cache_seconds, - ) - settings.append_exchange_rate_datapoint( - dict(rates), max_history_size - ) - except Exception as ex: - logger.warning(ex) + currency = settings.lnbits_default_accounting_currency or "USD" + max_history_size = settings.lnbits_exchange_history_size + try: + if ( + settings.lnbits_price_aggregator_enabled + and settings.lnbits_price_aggregator_url + ): + price = await btc_price_from_aggregator(currency) + if price: + cache.set( + f"btc-price-{currency}", + price, + expiry=settings.lnbits_exchange_rate_cache_seconds, + ) + settings.append_exchange_rate_datapoint( + {"Aggregator": price}, max_history_size + ) else: - sleep_time = 60 - await asyncio.sleep(sleep_time) + rates = await btc_rates(currency) + if rates: + rates_values = [r[1] for r in rates] + lnbits_rate = sum(rates_values) / len(rates_values) + rates.append(("LNbits", lnbits_rate)) + cache.set( + f"btc-price-{currency}", + lnbits_rate, + expiry=settings.lnbits_exchange_rate_cache_seconds, + ) + settings.append_exchange_rate_datapoint(dict(rates), max_history_size) + except Exception as ex: + logger.warning(ex) diff --git a/lnbits/core/views/admin_api.py b/lnbits/core/views/admin_api.py index 6e69d07c8..c46498b70 100644 --- a/lnbits/core/views/admin_api.py +++ b/lnbits/core/views/admin_api.py @@ -20,7 +20,7 @@ from lnbits.core.services.settings import dict_to_settings from lnbits.decorators import check_admin, check_super_user from lnbits.server import server_restart from lnbits.settings import AdminSettings, Settings, UpdateSettings, settings -from lnbits.tasks import invoice_listeners +from lnbits.task_manager import PublicTask, task_manager from .. import core_app_extra from ..crud import get_admin_settings, reset_core_settings, update_admin_settings @@ -44,11 +44,10 @@ async def api_auditor(): name="Monitor", description="show the current listeners and other monitoring data", dependencies=[Depends(check_admin)], + response_model=list[PublicTask], ) -async def api_monitor(): - return { - "invoice_listeners": list(invoice_listeners.keys()), - } +async def api_monitor() -> list[PublicTask]: + return task_manager.get_public_tasks() @admin_router.get( diff --git a/lnbits/settings.py b/lnbits/settings.py index 6ba128679..8d9bf2404 100644 --- a/lnbits/settings.py +++ b/lnbits/settings.py @@ -1077,11 +1077,12 @@ class EnvSettings(LNbitsSettings): log_rotation: str = Field(default="100 MB") log_retention: str = Field(default="3 months") first_install_token: str | None = Field(default=None) - cleanup_wallets_days: int = Field(default=90, ge=0) funding_source_max_retries: int = Field(default=4, ge=0) lnbits_max_users: int = Field(default=0, ge=0) lnbits_max_extensions: int = Field(default=0, ge=0) + task_heart_beat_verbose: bool = Field(default=False) + task_heart_beat_interval: int = Field(default=30) @property def has_default_extension_path(self) -> bool: diff --git a/lnbits/task_manager.py b/lnbits/task_manager.py new file mode 100644 index 000000000..6f72c9f83 --- /dev/null +++ b/lnbits/task_manager.py @@ -0,0 +1,200 @@ +import asyncio +import traceback +import uuid +from collections.abc import Callable, Coroutine +from datetime import datetime, timezone + +from loguru import logger +from pydantic import BaseModel + +from lnbits.core.models import Payment +from lnbits.settings import settings + + +class PublicTask(BaseModel): + """Public model used to expose task information via the API.""" + + name: str + created_at: datetime + + +class Task: + """Model used on the backend to keep track of background tasks.""" + + coro: Coroutine + name: str + created_at: datetime + task: asyncio.Task + invoice_queue: asyncio.Queue[Payment] | None = None + + def __init__( + self, + coro: Coroutine, + name: str | None = None, + invoice_queue: asyncio.Queue | None = None, + ) -> None: + self.coro = coro + self.name = name or f"task_{uuid.uuid4()}" + self.created_at = datetime.now(timezone.utc) + self.task = asyncio.create_task(self.coro, name=self.name) + self.invoice_queue = invoice_queue + + +class TaskManager: + """Singleton class to manage background tasks.""" + + tasks: list[Task] = [] + invoice_queue: asyncio.Queue[Payment] = asyncio.Queue() + internal_invoice_queue: asyncio.Queue[Payment] = asyncio.Queue() + + def init(self) -> None: + self.create_permanent_task( + func=self._heart_beat, + interval=settings.task_heart_beat_interval, + ) + self.create_permanent_task(self._invoice_listener_consumer) + self.create_permanent_task(self._internal_invoice_listener_consumer) + + def get_task(self, name: str) -> Task | None: + """Get a running task by name.""" + for task in self.tasks: + if task.name == name: + return task + return None + + def get_public_tasks(self) -> list[PublicTask]: + """Get a list of public tasks.""" + return [PublicTask(name=t.name, created_at=t.created_at) for t in self.tasks] + + def cancel_task(self, task: Task) -> None: + """Cancel a running task.""" + self.tasks.remove(task) + try: + task.task.cancel() + except Exception as exc: + logger.warning(f"error while cancelling task `{task.name}`: {exc!s}") + + def cancel_all_tasks(self) -> None: + """Cancel all running tasks.""" + for task in list(self.tasks): + self.cancel_task(task) + + def create_task( + self, + coro: Coroutine, + name: str | None = None, + invoice_queue: asyncio.Queue | None = None, + ) -> Task: + """Create a task. If a task with the same name exists, it will be cancelled.""" + if name: + task = self.get_task(name) + if task: + self.cancel_task(task) + task = Task(coro=coro, name=name, invoice_queue=invoice_queue) + self.tasks.append(task) + return task + + def create_permanent_task( + self, + func: Callable[[], Coroutine], + invoice_queue: asyncio.Queue | None = None, + name: str | None = None, + interval: int = 0, + ) -> Task: + """Create a task that runs forever and restarts on failure.""" + + async def wrapper(): + while settings.lnbits_running: + await self._catch_everything_and_restart(func) + if interval > 0: + await asyncio.sleep(interval) + + return self.create_task( + coro=wrapper(), name=name or func.__name__, invoice_queue=invoice_queue + ) + + def register_invoice_listener( + self, + func: Callable[[Payment], Coroutine], + name: str | None = None, + ) -> Task: + """ + A method intended for extensions to call when they want to be notified about + incoming payments. Will call provided Coroutine with the updated payment. + """ + name = f"{name or uuid.uuid4()}_invoice_listener" + queue: asyncio.Queue[Payment] = asyncio.Queue() + return self.create_permanent_task( + self._invoice_listener_worker(func, queue), + name=name, + invoice_queue=queue, + ) + + async def _heart_beat(self) -> None: + """A heartbeat that removes done tasks logs the number of tasks.""" + for task in self.tasks: + state = task.task._state if task.task else "NOT RUNNING" + if settings.task_heart_beat_verbose: + logger.debug( + f"Task Manager: `{task.name}` state: `{state}` " + f"created: {task.created_at.strftime('%Y-%m-%d %H:%M:%S')}`" + ) + if task.task and task.task.done(): + logger.debug(f"Task Manager: task `{task.name}` is done.") + self.cancel_task(task) + listeners_count = sum(1 for task in self.tasks if task.invoice_queue) + logger.debug( + f"Task Manager: {len(self.tasks) - listeners_count} tasks " + f"and {listeners_count} invoice listeners." + ) + + async def _catch_everything_and_restart( + self, + func: Callable[[], Coroutine], + restart_interval: int = 5, + ) -> None: + """Catches all exceptions from a function and restarts it after 5 seconds.""" + while settings.lnbits_running: + try: + return await func() + except asyncio.CancelledError: + raise # because we must pass this up + except Exception as exc: + if not settings.lnbits_running: + return + logger.error(f"exception in background task `{func.__name__}`:", exc) + logger.error(traceback.format_exc()) + logger.info( + f"`{func.__name__}` restarts in {restart_interval} seconds." + ) + await asyncio.sleep(restart_interval) + + def _invoice_listener_worker( + self, func: Callable[[Payment], Coroutine], queue: asyncio.Queue[Payment] + ) -> Callable: + async def wrapper() -> None: + payment: Payment = await queue.get() + await func(payment) + + return wrapper + + def _invoice_dispatcher(self, payment: Payment) -> None: + """Dispatches a payment to all registered invoice listeners.""" + for task in self.tasks: + if not task.invoice_queue: + continue + logger.debug(f"Enqueing payment to task {task.name}") + task.invoice_queue.put_nowait(payment) + + async def _invoice_listener_consumer(self) -> None: + payment = await self.invoice_queue.get() + logger.info(f"got a payment notification {payment.checking_id}") + self._invoice_dispatcher(payment) + + async def _internal_invoice_listener_consumer(self) -> None: + payment = await self.internal_invoice_queue.get() + logger.info(f"got an internal payment notification {payment.checking_id}") + self._invoice_dispatcher(payment) + + +task_manager = TaskManager() diff --git a/lnbits/tasks.py b/lnbits/tasks.py index 731e0cd03..6a02e44bf 100644 --- a/lnbits/tasks.py +++ b/lnbits/tasks.py @@ -1,146 +1,83 @@ import asyncio -import traceback import uuid from collections.abc import Callable, Coroutine from loguru import logger from lnbits.core.models import Payment -from lnbits.core.services.payments import ( - get_standalone_payment, - update_invoice_from_paid_invoices_stream, -) +from lnbits.core.services.payments import get_standalone_payment from lnbits.settings import settings -from lnbits.wallets import get_funding_source - -tasks: list[asyncio.Task] = [] -unique_tasks: dict[str, asyncio.Task] = {} +from lnbits.task_manager import task_manager +# DEPRECATED: use task_manager.create_task instead. def create_task(coro: Coroutine) -> asyncio.Task: - task = asyncio.create_task(coro) - tasks.append(task) - return task + logger.debug("DEPRECATED: use task_manager.create_task instead.") + return task_manager.create_task(coro).task +# DEPRECATED: use task_manager.create_task with `name` kwarg. def create_unique_task(name: str, coro: Coroutine) -> asyncio.Task: - if unique_tasks.get(name): - logger.warning(f"task `{name}` already exists, cancelling it") - try: - unique_tasks[name].cancel() - except Exception as exc: - logger.warning(f"error while cancelling task `{name}`: {exc!s}") - task = asyncio.create_task(coro) - unique_tasks[name] = task - return task + logger.debug("DEPRECATED: use task_manager.create_task instead.") + return task_manager.create_task(coro, name=name).task +# DEPRECATED: use task_manager.create_permanent_task instead. def create_permanent_task(func: Callable[[], Coroutine]) -> asyncio.Task: - return create_task(catch_everything_and_restart(func)) + logger.debug("DEPRECATED: use task_manager.create_permanent_task instead.") + return task_manager.create_permanent_task(func).task +# DEPRECATED: use task_manager.create_permanent_task with `name` argument instead. def create_permanent_unique_task( name: str, coro: Callable[[], Coroutine] ) -> asyncio.Task: return create_unique_task(name, catch_everything_and_restart(coro, name)) -def cancel_all_tasks() -> None: - for task in tasks: - try: - task.cancel() - except Exception as exc: - logger.warning(f"error while cancelling task: {exc!s}") - for name, task in unique_tasks.items(): - try: - task.cancel() - except Exception as exc: - logger.warning(f"error while cancelling task `{name}`: {exc!s}") - - +# DEPRECATED don't use this, use task_manager.create_permanent_task instead. async def catch_everything_and_restart( func: Callable[[], Coroutine], name: str = "unnamed", -) -> Coroutine: - try: - return await func() - except asyncio.CancelledError: - raise # because we must pass this up - except Exception as exc: - logger.error(f"exception in background task `{name}`:", exc) - logger.error(traceback.format_exc()) - logger.error("will restart the task in 5 seconds.") - await asyncio.sleep(5) - return await catch_everything_and_restart(func, name) +) -> None: + _ = name + return await task_manager._catch_everything_and_restart(func) -invoice_listeners: dict[str, asyncio.Queue] = {} - - -# TODO: name should not be optional -# some extensions still dont use a name, but they should def register_invoice_listener(send_chan: asyncio.Queue, name: str | None = None): """ - A method intended for extensions (and core/tasks.py) to call when they want to be - notified about new invoice payments incoming. Will emit all incoming payments. + DEPRECATED: use task_manager.register_invoice_listener instead, + which also allows to pass a callback instead of a queue. + This method will still work but it is not recommended for new code. """ - if not name: - # fallback to a random name if extension didn't provide one - name = f"no_name_{str(uuid.uuid4())[:8]}" + logger.debug("DEPRECATED: use task_manager.register_invoice_listener instead.") + name = f"forward_{name or str(uuid.uuid4())[:8]}" - if invoice_listeners.get(name): - logger.warning(f"invoice listener `{name}` already exists, replacing it") + # here we just forwarding the payments to the provided queue + async def forward_queue(payment: Payment): + send_chan.put_nowait(payment) - logger.trace(f"registering invoice listener `{name}`") - invoice_listeners[name] = send_chan - - -internal_invoice_queue: asyncio.Queue = asyncio.Queue(0) + task_manager.register_invoice_listener(forward_queue, name=name) async def internal_invoice_queue_put(checking_id: str) -> None: """ + DEPRECATED: use task_manager.internal_invoice_queue instead, A method to call when it wants to notify about an internal invoice payment. """ - await internal_invoice_queue.put(checking_id) - - -async def internal_invoice_listener() -> None: - """ - internal_invoice_queue will be filled directly in core/services.py - after the payment was deemed to be settled internally. - - Called by the app startup sequence. - """ - while settings.lnbits_running: - checking_id = await internal_invoice_queue.get() - logger.info(f"got an internal payment notification {checking_id}") - payment = await get_standalone_payment(checking_id, incoming=True) - if payment: - logger.success(f"internal invoice {checking_id} settled") - await invoice_callback_dispatcher(payment) - - -async def invoice_listener() -> None: - """ - invoice_listener will collect all invoices that come directly - from the backend wallet. - - Called by the app startup sequence. - """ - funding_source = get_funding_source() - async for checking_id in funding_source.paid_invoices_stream(): - logger.info(f"got a payment notification {checking_id}") - payment = await update_invoice_from_paid_invoices_stream(checking_id) - if payment: - logger.success(f"fundingsource invoice {checking_id} settled") - await invoice_callback_dispatcher(payment) + payment = await get_standalone_payment(checking_id, incoming=True) + if not payment: + logger.warning(f"internal_invoice_queue_put: payment {checking_id} not found") + return + await task_manager.internal_invoice_queue.put(payment) +# DEPRECATED use task_manager.register_invoice_listener(coro, name="myext") def wait_for_paid_invoices( invoice_listener_name: str, func: Callable[[Payment], Coroutine], ) -> Callable[[], Coroutine]: + logger.debug("DEPRECATED: use task_manager.register_invoice_listener instead.") async def wrapper() -> None: invoice_queue: asyncio.Queue = asyncio.Queue() @@ -150,27 +87,3 @@ def wait_for_paid_invoices( await func(payment) return wrapper - - -def run_interval( - interval_seconds: int, - func: Callable[[], Coroutine], -) -> Callable[[], Coroutine]: - """Run a function at a specified interval in seconds, while the server is running""" - - async def wrapper() -> None: - while settings.lnbits_running: - try: - await func() - except Exception as e: - logger.error(f"Error occurred in interval task: {e}") - logger.warning(traceback.format_exc()) - await asyncio.sleep(interval_seconds) - - return wrapper - - -async def invoice_callback_dispatcher(payment: Payment): - for name, send_chan in invoice_listeners.items(): - logger.trace(f"invoice listeners: sending to `{name}`") - await send_chan.put(payment) diff --git a/lnbits/utils/cache.py b/lnbits/utils/cache.py index 904447e19..9d8c6c3b1 100644 --- a/lnbits/utils/cache.py +++ b/lnbits/utils/cache.py @@ -1,13 +1,8 @@ from __future__ import annotations -import asyncio from time import time from typing import Any, NamedTuple -from loguru import logger - -from lnbits.settings import settings - class Cached(NamedTuple): value: Any @@ -22,8 +17,7 @@ class Cache: Small caching utility providing simple get/set interface (very much like redis) """ - def __init__(self, interval: float = 10) -> None: - self.interval = interval + def __init__(self) -> None: self._values: dict[Any, Cached] = {} def value(self, key: str) -> Cached | None: @@ -59,16 +53,11 @@ class Cache: self.set(key, value, expiry=expiry) return value - async def invalidate_forever(self): - while settings.lnbits_running: - try: - await asyncio.sleep(self.interval) - ts = time() - expired = [k for k, v in self._values.items() if v.expiry < ts] - for k in expired: - self._values.pop(k) - except Exception: - logger.error("Error invalidating cache") + async def invalidate_cache(self): + ts = time() + expired = [k for k, v in self._values.items() if v.expiry < ts] + for k in expired: + self._values.pop(k) cache = Cache() diff --git a/lnbits/utils/logger.py b/lnbits/utils/logger.py index 0d78e4414..06c80822d 100644 --- a/lnbits/utils/logger.py +++ b/lnbits/utils/logger.py @@ -41,19 +41,16 @@ def log_server_info(): def initialize_server_websocket_logger() -> Callable: super_user_hash = sha256(settings.super_user.encode("utf-8")).hexdigest() - serverlog_queue: asyncio.Queue = asyncio.Queue() - - async def update_websocket_serverlog(): - while settings.lnbits_running: - msg = await serverlog_queue.get() - await websocket_updater(super_user_hash, msg) - logger.add( lambda msg: serverlog_queue.put_nowait(msg), format=Formatter().format, ) + async def update_websocket_serverlog(): + msg = await serverlog_queue.get() + await websocket_updater(super_user_hash, msg) + return update_websocket_serverlog diff --git a/tests/api/test_admin_api.py b/tests/api/test_admin_api.py index 39f8ad09c..c5537d665 100644 --- a/tests/api/test_admin_api.py +++ b/tests/api/test_admin_api.py @@ -82,7 +82,8 @@ async def test_admin_audit_monitor_and_test_email( headers={"Authorization": f"Bearer {superuser_token}"}, ) assert monitor.status_code == 200 - assert "invoice_listeners" in monitor.json() + task_names = [t["name"] for t in monitor.json()] + assert any("invoice_listener" in name for name in task_names) test_email = await client.get( "/admin/api/v1/testemail", diff --git a/tests/regtest/test_real_invoice.py b/tests/regtest/test_real_invoice.py index cb65b9937..6fbed901a 100644 --- a/tests/regtest/test_real_invoice.py +++ b/tests/regtest/test_real_invoice.py @@ -13,10 +13,13 @@ from lnbits.core.services import ( fee_reserve_total, get_balance_delta, ) -from lnbits.core.services.payments import pay_invoice, update_wallet_balance +from lnbits.core.services.payments import ( + pay_invoice, + update_wallet_balance, +) from lnbits.core.services.users import create_user_account from lnbits.exceptions import PaymentError -from lnbits.tasks import create_task, wait_for_paid_invoices +from lnbits.task_manager import task_manager from lnbits.wallets import get_funding_source from ..helpers import is_fake, is_regtest @@ -160,12 +163,11 @@ async def test_create_real_invoice( assert not payment_status["paid"] on_paid_mock = mocker.AsyncMock() - create_task(wait_for_paid_invoices("test_create_invoice", on_paid_mock)()) + task_manager.register_invoice_listener(on_paid_mock, "test_create_invoice") pay_real_invoice(invoice["bolt11"]) await asyncio.sleep(1) - assert on_paid_mock.call_count == 1 payment = on_paid_mock.call_args_list[0][0][0] @@ -393,12 +395,11 @@ async def test_receive_real_invoice_set_pending_and_check_state( assert not payment_status["paid"] on_paid_mock = mocker.AsyncMock() - create_task(wait_for_paid_invoices("test_create_invoice", on_paid_mock)()) + task_manager.register_invoice_listener(on_paid_mock, "test_create_invoice") pay_real_invoice(invoice["bolt11"]) await asyncio.sleep(1) - assert on_paid_mock.call_count == 1 payment = on_paid_mock.call_args_list[0][0][0] @@ -412,6 +413,8 @@ async def test_receive_real_invoice_set_pending_and_check_state( payment_status = response.json() assert payment_status["paid"] + assert payment + # set the incoming invoice to pending payment.status = PaymentState.PENDING await update_payment(payment) diff --git a/tests/unit/test_cache.py b/tests/unit/test_cache.py index 2f33ffe1d..d08a1c03d 100644 --- a/tests/unit/test_cache.py +++ b/tests/unit/test_cache.py @@ -5,6 +5,7 @@ import pytest from pytest_mock.plugin import MockerFixture from lnbits.settings import Settings +from lnbits.task_manager import task_manager from lnbits.utils.cache import Cache, Cached key = "foo" @@ -13,11 +14,10 @@ value = "bar" @pytest.fixture async def cache(): - cache = Cache(interval=0.1) - - task = asyncio.create_task(cache.invalidate_forever()) + cache = Cache() + task = task_manager.create_permanent_task(cache.invalidate_cache, interval=1) yield cache - task.cancel() + task_manager.cancel_task(task) @pytest.mark.anyio @@ -31,13 +31,13 @@ async def test_cache_get_set(cache): @pytest.mark.anyio async def test_cache_expiry(cache): # gets expired by `get` call - cache.set(key, value, expiry=0.01) - await asyncio.sleep(0.02) + cache.set(key, value, expiry=1) + await asyncio.sleep(2) assert not cache.get(key) # gets expired by invalidation task - cache.set(key, value, expiry=0.1) - await asyncio.sleep(0.2) + cache.set(key, value, expiry=1) + await asyncio.sleep(2) assert key not in cache._values assert not cache.get(key) @@ -94,23 +94,33 @@ async def test_cache_pop_expired_returns_default(cache): async def test_invalidate_forever_logs_and_recovers_from_errors( settings: Settings, mocker: MockerFixture ): - test_cache = Cache(interval=0) - logger_error = mocker.patch("lnbits.utils.cache.logger.error") + test_cache = Cache() original_running = settings.lnbits_running calls = 0 - async def fake_sleep(_interval): + original_invalidate = test_cache.invalidate_cache + + async def fake_invalidate(): nonlocal calls calls += 1 if calls == 1: raise RuntimeError("boom") settings.lnbits_running = False + await original_invalidate() + mocker.patch.object(test_cache, "invalidate_cache", side_effect=fake_invalidate) + mocker.patch("lnbits.task_manager.asyncio.sleep") + logger_error = mocker.patch("lnbits.task_manager.logger.error") + + bg_task = None try: settings.lnbits_running = True - mocker.patch("lnbits.utils.cache.asyncio.sleep", side_effect=fake_sleep) - await test_cache.invalidate_forever() + bg_task = task_manager.create_permanent_task(test_cache.invalidate_cache) + await bg_task.task finally: settings.lnbits_running = original_running + if bg_task: + task_manager.cancel_task(bg_task) - logger_error.assert_called_once_with("Error invalidating cache") + assert logger_error.called + assert calls == 2 diff --git a/tests/unit/test_fiat_providers.py b/tests/unit/test_fiat_providers.py index 995371997..681761449 100644 --- a/tests/unit/test_fiat_providers.py +++ b/tests/unit/test_fiat_providers.py @@ -1717,7 +1717,9 @@ async def test_check_fiat_status_handles_internal_states(mocker: MockerFixture): "lnbits.core.services.fiat_providers.get_fiat_provider", AsyncMock(return_value=provider), ) - queue_put = mocker.patch("lnbits.tasks.internal_invoice_queue.put", AsyncMock()) + queue_put = mocker.patch( + "lnbits.task_manager.task_manager.internal_invoice_queue.put_nowait" + ) success_status = await check_fiat_status( Payment( @@ -1734,7 +1736,8 @@ async def test_check_fiat_status_handles_internal_states(mocker: MockerFixture): ) assert success_status.success is True - queue_put.assert_awaited_once_with("fiat_pending") + queue_put.assert_called_once() + assert queue_put.call_args[0][0].checking_id == "fiat_pending" await check_fiat_status( Payment( @@ -1749,7 +1752,7 @@ async def test_check_fiat_status_handles_internal_states(mocker: MockerFixture): extra={"fiat_checking_id": "stripe_checking_id"}, ) ) - assert queue_put.await_count == 1 + assert queue_put.call_count == 1 @pytest.mark.anyio @@ -1786,7 +1789,9 @@ async def test_check_fiat_status_persists_successful_payment( "lnbits.fiat.StripeWallet.get_invoice_status", AsyncMock(return_value=FiatPaymentStatus(paid=True)), ) - queue_put = mocker.patch("lnbits.tasks.internal_invoice_queue.put", AsyncMock()) + queue_put = mocker.patch( + "lnbits.task_manager.task_manager.internal_invoice_queue.put_nowait" + ) status = await check_fiat_status(payment) @@ -1794,7 +1799,7 @@ async def test_check_fiat_status_persists_successful_payment( assert payment.status == PaymentState.SUCCESS updated_payment = await get_payment(payment.checking_id) assert updated_payment.status == PaymentState.SUCCESS - queue_put.assert_awaited_once_with(payment.checking_id) + queue_put.assert_called_once_with(payment) @pytest.mark.anyio diff --git a/tests/unit/test_pay_invoice.py b/tests/unit/test_pay_invoice.py index 070e2e70c..eaa481daf 100644 --- a/tests/unit/test_pay_invoice.py +++ b/tests/unit/test_pay_invoice.py @@ -12,15 +12,12 @@ from lnbits.core.crud import create_wallet, get_standalone_payment, get_wallet from lnbits.core.crud.payments import get_payment, get_payments_paginated from lnbits.core.models import PaymentState, Wallet from lnbits.core.services import create_invoice, create_user_account, pay_invoice -from lnbits.core.services.payments import update_wallet_balance +from lnbits.core.services.payments import ( + update_wallet_balance, +) from lnbits.exceptions import InvoiceError, PaymentError from lnbits.settings import Settings -from lnbits.tasks import ( - create_task, - internal_invoice_listener, - internal_invoice_queue, - wait_for_paid_invoices, -) +from lnbits.task_manager import task_manager from lnbits.wallets.base import PaymentResponse from lnbits.wallets.fake import FakeWallet @@ -237,24 +234,30 @@ async def test_notification_for_internal_payment( test_name = "test_notification_for_internal_payment" # Drain stale items left by session-scoped fixtures (e.g. update_wallet_balance) - while not internal_invoice_queue.empty(): + while not task_manager.internal_invoice_queue.empty(): try: - internal_invoice_queue.get_nowait() + task_manager.internal_invoice_queue.get_nowait() except asyncio.QueueEmpty: break on_paid_mock = mocker.AsyncMock() - create_task(internal_invoice_listener()) - create_task(wait_for_paid_invoices(test_name, on_paid_mock)()) + # create_task(internal_invoice_listener()) + + task_manager.register_invoice_listener(on_paid_mock, test_name) + payment = await create_invoice( wallet_id=to_wallet.id, amount=123, memo=test_name, webhook="http://test.404.lnbits.com", ) - await pay_invoice( + paid_payment = await pay_invoice( wallet_id=to_wallet.id, payment_request=payment.bolt11, extra={"tag": "lnurlp"} ) + assert paid_payment.status == PaymentState.SUCCESS.value + assert paid_payment.bolt11 == payment.bolt11 + assert paid_payment.amount == -123_000 + await asyncio.sleep(1) assert on_paid_mock.call_count == 1 @@ -264,6 +267,8 @@ async def test_notification_for_internal_payment( assert _payment.status == PaymentState.SUCCESS.value assert _payment.bolt11 == payment.bolt11 assert _payment.amount == 123_000 + assert _payment.checking_id == payment.checking_id + updated_payment = await get_payment(_payment.checking_id) assert ( updated_payment.webhook_status is not None diff --git a/tests/unit/test_services_payments.py b/tests/unit/test_services_payments.py index c847363eb..a3ad69042 100644 --- a/tests/unit/test_services_payments.py +++ b/tests/unit/test_services_payments.py @@ -197,8 +197,7 @@ async def test_update_wallet_balance_validates_credit_and_debit( settings.lnbits_wallet_limit_max_balance = 0 queue_mock = mocker.patch( - "lnbits.tasks.internal_invoice_queue_put", - mocker.AsyncMock(), + "lnbits.task_manager.task_manager.internal_invoice_queue.put_nowait", ) await update_wallet_balance(wallet, 5) @@ -212,7 +211,8 @@ async def test_update_wallet_balance_validates_credit_and_debit( ] assert credit_payments assert credit_payments[0].status == PaymentState.SUCCESS - queue_mock.assert_awaited_once_with(credit_payments[0].checking_id) + queue_mock.assert_called_once() + assert queue_mock.call_args[0][0].checking_id == credit_payments[0].checking_id @pytest.mark.anyio