[feat] ui support for high number of wallets and payments (#3174)

lnbits/core/crud/payments.py

@@ -123,12 +123,8 @@ async def get_payments_paginated(
         values["wallet_id"] = wallet_id
         clause.append("wallet_id = :wallet_id")
     elif user_id:
-        wallet_ids = await get_wallets_ids(user_id=user_id, conn=conn) or [
-            "no-wallets-for-user"
-        ]
-        # wallet ids are safe to use in sql queries
-        wallet_ids_str = [f"'{w}'" for w in wallet_ids]
-        clause.append(f""" wallet_id IN ({", ".join(wallet_ids_str)}) """)
+        only_user_wallets = await _only_user_wallets_statement(user_id, conn=conn)
+        clause.append(only_user_wallets)
 
     if complete and pending:
         clause.append(
@@ -366,12 +362,19 @@ async def get_payments_history(
 async def get_payment_count_stats(
     field: PaymentCountField,
     filters: Optional[Filters[PaymentFilters]] = None,
+    user_id: Optional[str] = None,
     conn: Optional[Connection] = None,
 ) -> list[PaymentCountStat]:
 
     if not filters:
         filters = Filters()
-    clause = filters.where()
+    extra_stmts = []
+
+    if user_id:
+        only_user_wallets = await _only_user_wallets_statement(user_id, conn=conn)
+        extra_stmts.append(only_user_wallets)
+
+    clause = filters.where(extra_stmts)
     data = await (conn or db).fetchall(
         query=f"""
             SELECT {field} as field, count(*) as total
@@ -389,18 +392,26 @@ async def get_payment_count_stats(
 
 async def get_daily_stats(
     filters: Optional[Filters[PaymentFilters]] = None,
+    user_id: Optional[str] = None,
     conn: Optional[Connection] = None,
 ) -> Tuple[list[PaymentDailyStats], list[PaymentDailyStats]]:
 
     if not filters:
         filters = Filters()
 
-    in_clause = filters.where(
-        ["(apipayments.status = 'success' AND apipayments.amount > 0)"]
-    )
-    out_clause = filters.where(
-        ["(apipayments.status IN ('success', 'pending') AND apipayments.amount < 0)"]
-    )
+    in_where_stmts = ["(apipayments.status = 'success' AND apipayments.amount > 0)"]
+    out_where_stmts = [
+        "(apipayments.status IN ('success', 'pending') AND apipayments.amount < 0)"
+    ]
+
+    if user_id:
+        only_user_wallets = await _only_user_wallets_statement(user_id, conn=conn)
+        in_where_stmts.append(only_user_wallets)
+        out_where_stmts.append(only_user_wallets)
+
+    in_clause = filters.where(in_where_stmts)
+    out_clause = filters.where(out_where_stmts)
+
     date_trunc = db.datetime_grouping("day")
     query = """
         SELECT {date_trunc} date,
@@ -431,6 +442,7 @@ async def get_daily_stats(
 
 async def get_wallets_stats(
     filters: Optional[Filters[PaymentFilters]] = None,
+    user_id: Optional[str] = None,
     conn: Optional[Connection] = None,
 ) -> list[PaymentWalletStats]:
 
@@ -449,6 +461,10 @@ async def get_wallets_stats(
         )
         """,
     ]
+    if user_id:
+        only_user_wallets = await _only_user_wallets_statement(user_id, conn=conn)
+        where_stmts.append(only_user_wallets)
+
     clauses = filters.where(where_stmts)
 
     data = await (conn or db).fetchall(
@@ -524,3 +540,14 @@ async def mark_webhook_sent(payment_hash: str, status: str) -> None:
         """,
         {"status": status, "hash": payment_hash},
     )
+
+
+async def _only_user_wallets_statement(
+    user_id: str, conn: Optional[Connection] = None
+) -> str:
+    wallet_ids = await get_wallets_ids(user_id=user_id, conn=conn) or [
+        "no-wallets-for-user"
+    ]
+    # wallet ids are safe to use in sql queries
+    wallet_ids_str = [f"'{w}'" for w in wallet_ids]
+    return f""" wallet_id IN ({", ".join(wallet_ids_str)}) """

lnbits/core/crud/wallets.py

@@ -4,7 +4,8 @@ from typing import Optional
 from uuid import uuid4
 
 from lnbits.core.db import db
-from lnbits.db import Connection
+from lnbits.core.models.wallets import WalletsFilters
+from lnbits.db import Connection, Filters, Page
 from lnbits.settings import settings
 
 from ..models import Wallet
@@ -135,6 +136,29 @@ async def get_wallets(
     )
 
 
+async def get_wallets_paginated(
+    user_id: str,
+    deleted: Optional[bool] = None,
+    filters: Optional[Filters[WalletsFilters]] = None,
+    conn: Optional[Connection] = None,
+) -> Page[Wallet]:
+    if deleted is None:
+        deleted = False
+
+    where: list[str] = [""" "user" = :user AND deleted = :deleted """]
+    return await (conn or db).fetch_page(
+        """
+            SELECT *, COALESCE((
+                SELECT balance FROM balances WHERE wallet_id = wallets.id
+            ), 0) AS balance_msat FROM wallets
+        """,
+        where=where,
+        values={"user": user_id, "deleted": deleted},
+        filters=filters,
+        model=Wallet,
+    )
+
+
 async def get_wallets_ids(
     user_id: str, deleted: Optional[bool] = None, conn: Optional[Connection] = None
 ) -> list[str]: