Improved SSE listeners (#865)

* logging listeners

* comments

* generate privkey upon init

* listener queue

* remove duplicate check

* make format

* reuse channel

* error handling in sse listener

* uuid for listeners

* register named invoices

* uuid for listeners and listener list

* fix poetry lock

* setuptools

* requirements asyncio timeout

* setuptool;s

* make format

* remove async-timeout

* async_timeout readd

* try lower setuptools version

* try lower lower setuptools version

* back to current version + fix, maybe

* fix worflows to use poetry 1.2.1

* remove uneeded setuptools from build-system

* fix up formatting workflow

* debug to trace

* more traces

* debug logs to trace

Co-authored-by: dni ⚡ <office@dnilabs.com>

lnbits/core/tasks.py

@@ -1,30 +1,43 @@
 import asyncio
-from typing import List
+from typing import Dict
 
 import httpx
 from loguru import logger
 
-from lnbits.tasks import register_invoice_listener
+from lnbits.helpers import get_current_extension_name
+from lnbits.tasks import SseListenersDict, register_invoice_listener
 
 from . import db
 from .crud import get_balance_notify
 from .models import Payment
 
-api_invoice_listeners: List[asyncio.Queue] = []
+api_invoice_listeners: Dict[str, asyncio.Queue] = SseListenersDict(
+    "api_invoice_listeners"
+)
 
 
 async def register_task_listeners():
+    """
+    Registers an invoice listener queue for the core tasks.
+    Incoming payaments in this queue will eventually trigger the signals sent to all other extensions
+    and fulfill other core tasks such as dispatching webhooks.
+    """
     invoice_paid_queue = asyncio.Queue(5)
-    register_invoice_listener(invoice_paid_queue)
+    # we register invoice_paid_queue to receive all incoming invoices
+    register_invoice_listener(invoice_paid_queue, "core/tasks.py")
+    # register a worker that will react to invoices
     asyncio.create_task(wait_for_paid_invoices(invoice_paid_queue))
 
 
 async def wait_for_paid_invoices(invoice_paid_queue: asyncio.Queue):
+    """
+    This worker dispatches events to all extensions, dispatches webhooks and balance notifys.
+    """
     while True:
         payment = await invoice_paid_queue.get()
-        logger.debug("received invoice paid event")
+        logger.trace("received invoice paid event")
         # send information to sse channel
-        await dispatch_invoice_listener(payment)
+        await dispatch_api_invoice_listeners(payment)
 
         # dispatch webhook
         if payment.webhook and not payment.webhook_status:
@@ -41,16 +54,23 @@ async def wait_for_paid_invoices(invoice_paid_queue: asyncio.Queue):
                     pass
 
 
-async def dispatch_invoice_listener(payment: Payment):
-    for send_channel in api_invoice_listeners:
+async def dispatch_api_invoice_listeners(payment: Payment):
+    """
+    Emits events to invoice listener subscribed from the API.
+    """
+    for chan_name, send_channel in api_invoice_listeners.items():
         try:
+            logger.debug(f"sending invoice paid event to {chan_name}")
             send_channel.put_nowait(payment)
         except asyncio.QueueFull:
-            logger.debug("removing sse listener", send_channel)
-            api_invoice_listeners.remove(send_channel)
+            logger.error(f"removing sse listener {send_channel}:{chan_name}")
+            api_invoice_listeners.pop(chan_name)
 
 
 async def dispatch_webhook(payment: Payment):
+    """
+    Dispatches the webhook to the webhook url.
+    """
     async with httpx.AsyncClient() as client:
         data = payment.dict()
         try: