feat: introduce external_id

lnbits/core/crud/payments.py

@@ -107,35 +107,6 @@ async def get_latest_payments_by_extension(
     )
 
 
-async def get_latest_payment_by_extra_key_value(
-    key: str,
-    value: str,
-    wallet_id: str | None = None,
-    conn: Connection | None = None,
-) -> Payment | None:
-    values = {
-        "extra_key": f'%"{key}"%',
-        "extra_value": f'%"{value}"%',
-    }
-    clause = ["extra LIKE :extra_key", "extra LIKE :extra_value"]
-    if wallet_id:
-        wallet = await get_wallet(wallet_id, conn=conn)
-        if not wallet or not wallet.can_view_payments:
-            return None
-        values["wallet_id"] = wallet.source_wallet_id
-        clause.append("wallet_id = :wallet_id")
-
-    return await (conn or db).fetchone(
-        f"""
-        SELECT * FROM apipayments
-        WHERE {" AND ".join(clause)}
-        ORDER BY time DESC LIMIT 1
-        """,  # noqa: S608
-        values,
-        Payment,
-    )
-
-
 async def get_payments_paginated(  # noqa: C901
     *,
     wallet_id: str | None = None,
@@ -321,6 +292,7 @@ async def create_payment(
         tag=extra.get("tag", None),
         extra=extra,
         labels=data.labels or [],
+        external_id=data.external_id,
     )
 
     await (conn or db).insert("apipayments", payment)

lnbits/core/migrations.py

@@ -802,3 +802,16 @@ async def m044_add_activated_to_accounts(db: Connection):
     Used for account activation status.
     """
     await db.execute("ALTER TABLE accounts ADD COLUMN activated BOOLEAN DEFAULT true")
+
+
+async def m045_add_external_id_to_payments(db: Connection):
+    """
+    Adds external_id column to apipayments.
+    Used for external payment references.
+    """
+    await db.execute("ALTER TABLE apipayments ADD COLUMN external_id TEXT")
+    logger.debug("Creating index idx_payments_external_id...")
+    await db.execute("""
+        CREATE INDEX IF NOT EXISTS idx_payments_external_id
+        ON apipayments (external_id);
+        """)

lnbits/core/models/payments.py

@@ -13,6 +13,7 @@ from lnbits.db import FilterModel
 from lnbits.fiat.base import (
     FiatPaymentStatus,
 )
+from lnbits.helpers import is_valid_external_id
 from lnbits.utils.exchange_rates import allowed_currencies
 from lnbits.wallets.base import (
     PaymentStatus,
@@ -53,6 +54,11 @@ class CreatePayment(BaseModel):
     webhook: str | None = None
     fee: int = 0
     labels: list[str] | None = None
+    external_id: str | None = None
+
+    @validator("external_id")
+    def validate_external_id(cls, external_id):
+        return _validate_external_id(external_id)
 
 
 class Payment(BaseModel):
@@ -77,6 +83,11 @@ class Payment(BaseModel):
     updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
     labels: list[str] = []
     extra: dict = {}
+    external_id: str | None = None
+
+    @validator("external_id")
+    def validate_external_id(cls, external_id):
+        return _validate_external_id(external_id)
 
     def __init__(self, **data):
         super().__init__(**data)
@@ -151,6 +162,7 @@ class PaymentFilters(FilterModel):
         "status",
         "time",
         "labels",
+        "external_id",
     ]
 
     __sort_fields__ = [
@@ -161,11 +173,13 @@ class PaymentFilters(FilterModel):
         "memo",
         "time",
         "tag",
+        "external_id",
     ]
 
     status: str | None
     tag: str | None
     checking_id: str | None
+    external_id: str | None
     amount: int
     fee: int
     memo: str | None
@@ -249,6 +263,7 @@ class CreateInvoice(BaseModel):
     lnurl_withdraw: LnurlWithdrawResponse | None = None
     fiat_provider: str | None = None
     labels: list[str] = []
+    external_id: str | None = Query(default=None, max_length=256)
 
     @validator("payment_hash")
     def check_hex(cls, v):
@@ -263,6 +278,10 @@ class CreateInvoice(BaseModel):
             raise ValueError("The provided unit is not supported")
         return v
 
+    @validator("external_id")
+    def validate_external_id(cls, external_id):
+        return _validate_external_id(external_id)
+
 
 class PaymentsStatusCount(BaseModel):
     incoming: int = 0
@@ -301,3 +320,12 @@ class CancelInvoice(BaseModel):
 
 class UpdatePaymentLabels(BaseModel):
     labels: list[str] = []
+
+
+def _validate_external_id(external_id: str | None) -> str | None:
+    if external_id and not is_valid_external_id(external_id):
+        raise ValueError(
+            "Invalid external id. Max length is 256 characters. "
+            "Space and newlines are not allowed."
+        )
+    return external_id

lnbits/core/services/payments.py

@@ -64,6 +64,7 @@ async def pay_invoice(
     description: str = "",
     tag: str = "",
     labels: list[str] | None = None,
+    external_id: str | None = None,
     conn: Connection | None = None,
 ) -> Payment:
     if settings.lnbits_only_allow_incoming_payments:
@@ -97,6 +98,7 @@ async def pay_invoice(
             memo=description or invoice.description or "",
             extra=extra,
             labels=labels,
+            external_id=external_id,
         )
 
     async with db.reuse_conn(conn) if conn else db.connect() as new_conn:
@@ -217,6 +219,7 @@ async def create_wallet_invoice(wallet_id: str, data: CreateInvoice) -> Payment:
             internal=data.internal,
             payment_hash=data.payment_hash,
             labels=data.labels,
+            external_id=data.external_id,
             conn=conn,
         )
 
@@ -258,6 +261,7 @@ async def create_invoice(
     internal: bool | None = False,
     payment_hash: str | None = None,
     labels: list[str] | None = None,
+    external_id: str | None = None,
     conn: Connection | None = None,
 ) -> Payment:
     if not amount > 0:
@@ -342,6 +346,7 @@ async def create_invoice(
         webhook=webhook,
         fee=invoice_response.fee_msat or 0,
         labels=labels,
+        external_id=external_id,
     )
 
     payment = await create_payment(

lnbits/core/views/callback_api.py

@@ -4,11 +4,11 @@ from fastapi import APIRouter, Request
 from loguru import logger
 
 from lnbits.core.crud.payments import (
-    get_latest_payment_by_extra_key_value,
+    get_payments,
     get_standalone_payment,
     update_payment,
 )
-from lnbits.core.models import Payment
+from lnbits.core.models import Payment, PaymentFilters
 from lnbits.core.models.misc import SimpleStatus
 from lnbits.core.models.payments import CreateInvoice
 from lnbits.core.services.fiat_providers import (
@@ -18,6 +18,7 @@ from lnbits.core.services.fiat_providers import (
     verify_paypal_webhook,
 )
 from lnbits.core.services.payments import create_fiat_invoice
+from lnbits.db import Filter, Filters
 from lnbits.fiat import get_fiat_provider
 from lnbits.fiat.base import FiatSubscriptionPaymentOptions
 from lnbits.fiat.square import SquareWallet
@@ -364,13 +365,27 @@ async def _handle_square_invoice_payment_made(event: dict):
 
     payment_options = _deserialize_square_metadata(_square_payment_note(payment))
     if not payment_options.wallet_id:
+        payment_id = payment.get("id")
         stored_payment = (
-            await get_latest_payment_by_extra_key_value(
+            await get_standalone_payment(f"fiat_square_payment_{payment_id}")
-                "square_subscription_id", subscription_id
+            if payment_id
-            )
-            if subscription_id
             else None
         )
+        if not stored_payment and subscription_id:
+            stored_payments = await get_payments(
+                filters=Filters(
+                    filters=[
+                        Filter.parse_query(
+                            "external_id", [subscription_id], PaymentFilters
+                        )
+                    ],
+                    model=PaymentFilters,
+                    sortby="created_at",
+                    direction="desc",
+                    limit=1,
+                )
+            )
+            stored_payment = stored_payments[0] if stored_payments else None
         if stored_payment:
             payment_options = _square_payment_options_from_payment(stored_payment)
         else:
@@ -406,10 +421,9 @@ async def _handle_square_subscription_payment(
     if existing_payment:
         if (
             square_subscription_id
-            and (existing_payment.extra or {}).get("square_subscription_id")
+            and existing_payment.external_id != square_subscription_id
-            != square_subscription_id
         ):
-            existing_payment.extra["square_subscription_id"] = square_subscription_id
+            existing_payment.external_id = square_subscription_id
             await update_payment(existing_payment)
         await check_fiat_status(existing_payment)
         return
@@ -427,8 +441,6 @@ async def _handle_square_subscription_payment(
             "payment_request": payment_request,
         },
     }
-    if square_subscription_id:
-        extra["square_subscription_id"] = square_subscription_id
 
     lnbits_payment = await create_fiat_invoice(
         wallet_id=wallet_id,
@@ -438,6 +450,7 @@ async def _handle_square_subscription_payment(
             memo=payment_options.memo or "",
             extra=extra,
             fiat_provider="square",
+            external_id=square_subscription_id,
         ),
     )
 
@@ -493,6 +506,5 @@ def _deserialize_square_metadata(custom_id: str) -> FiatSubscriptionPaymentOptio
             extra=extra,
             memo=memo,
         )
-    except (json.JSONDecodeError, IndexError, TypeError) as e:
+    except (json.JSONDecodeError, IndexError, TypeError):
-        logger.debug(f"Failed to deserialize Square metadata: {e}")
         return FiatSubscriptionPaymentOptions()

lnbits/core/views/payment_api.py

@@ -263,6 +263,7 @@ async def api_payments_create(
             payment_request=invoice_data.bolt11,
             extra=invoice_data.extra,
             labels=invoice_data.labels,
+            external_id=invoice_data.external_id,
         )
         return payment
 

lnbits/fiat/square.py

@@ -312,6 +312,7 @@ class SquareWallet(FiatProvider):
     ) -> SquareSubscriptionCheckoutInfo:
         plan_data = catalog_object.get("subscription_plan_data") or {}
         plan_variations = plan_data.get("subscription_plan_variations") or []
+        eligible_item_ids = plan_data.get("eligible_item_ids") or []
         plan_variation = next(
             (
                 variation
@@ -325,7 +326,7 @@ class SquareWallet(FiatProvider):
 
         price_money = await self._get_subscription_price_money(
             plan_variation,
-            eligible_item_ids=plan_data.get("eligible_item_ids") or [],
+            eligible_item_ids=eligible_item_ids,
         )
         plan_variation_id = plan_variation.get("id")
         if not plan_variation_id:
@@ -547,15 +548,57 @@ class SquareWallet(FiatProvider):
     async def _get_square_subscription_id(
         self, subscription_id: str, wallet_id: str
     ) -> str:
-        from lnbits.core.crud.payments import get_latest_payment_by_extra_key_value
+        from lnbits.core.crud.payments import get_payments
+        from lnbits.core.models import PaymentFilters
+        from lnbits.db import Filter, Filters
 
-        payment = await get_latest_payment_by_extra_key_value(
+        payments = await get_payments(
-            "subscription_request_id", subscription_id, wallet_id=wallet_id
+            wallet_id=wallet_id,
+            filters=Filters(
+                filters=[
+                    Filter.parse_query("external_id", [subscription_id], PaymentFilters)
+                ],
+                model=PaymentFilters,
+                sortby="created_at",
+                direction="desc",
+                limit=1,
+            ),
         )
-        if not payment:
+        payment = next(
-            return subscription_id
+            (
-        square_subscription_id = (payment.extra or {}).get("square_subscription_id")
+                payment
-        return square_subscription_id or subscription_id
+                for payment in payments
+                if payment.external_id and payment.fiat_provider == "square"
+            ),
+            None,
+        )
+        if payment and payment.external_id:
+            return payment.external_id
+
+        payments = await get_payments(
+            wallet_id=wallet_id,
+            incoming=True,
+            filters=Filters(
+                model=PaymentFilters,
+                sortby="created_at",
+                direction="desc",
+            ),
+        )
+        payment = next(
+            (
+                payment
+                for payment in payments
+                if payment.external_id
+                and payment.fiat_provider == "square"
+                and (payment.extra or {}).get("subscription_request_id")
+                == subscription_id
+            ),
+            None,
+        )
+        if payment and payment.external_id:
+            return payment.external_id
+
+        return subscription_id
 
     def _settings_connection_fields(self) -> str:
         return "-".join(

tests/api/test_callback_api.py

@@ -4,7 +4,7 @@ from uuid import uuid4
 import pytest
 from httpx import AsyncClient
 
-from lnbits.core.models import Account, CreateInvoice
+from lnbits.core.models import Account, CreateInvoice, Payment
 from lnbits.core.services.payments import create_wallet_invoice
 from lnbits.core.services.users import create_user_account
 from lnbits.core.views.callback_api import (
@@ -253,9 +253,9 @@ async def test_callback_api_handles_subscription_flows_and_validation(
     payment.extra = {
         "subscription_request_id": "subscription_square_1",
         "tag": "members",
-        "square_subscription_id": "SUBSCRIPTION_1",
         "link": "link-1",
     }
+    payment.external_id = "SUBSCRIPTION_1"
     payment.memo = "Square Members"
     settings.square_api_endpoint = "https://connect.squareupsandbox.com"
     settings.square_access_token = "square-token"
@@ -276,8 +276,8 @@ async def test_callback_api_handles_subscription_flows_and_validation(
         mocker.AsyncMock(return_value=square_provider),
     )
     mocker.patch(
-        "lnbits.core.views.callback_api.get_latest_payment_by_extra_key_value",
+        "lnbits.core.views.callback_api.get_payments",
-        mocker.AsyncMock(return_value=payment),
+        mocker.AsyncMock(return_value=[payment]),
     )
 
     await handle_square_event(
@@ -298,7 +298,8 @@ async def test_callback_api_handles_subscription_flows_and_validation(
     assert create_fiat_invoice_mock.await_count == 4
     square_invoice_call = create_fiat_invoice_mock.await_args.kwargs
     square_invoice = square_invoice_call["invoice_data"]
-    assert square_invoice.extra["square_subscription_id"] == "SUBSCRIPTION_1"
+    assert square_invoice.external_id == "SUBSCRIPTION_1"
+    assert "square_subscription_id" not in square_invoice.extra
     assert (
         square_invoice.extra["subscription"]["payment_request"]
         == "https://square.example/invoice"
@@ -314,3 +315,83 @@ async def test_callback_api_handles_subscription_flows_and_validation(
                 "resource": {"amount": {"currency": "USD", "total": "5.00"}},
             }
         )
+
+
+@pytest.mark.anyio
+async def test_square_invoice_payment_updates_existing_subscription_external_id(
+    settings: Settings, mocker
+):
+    payment = Payment(
+        checking_id="fiat_square_payment_PAYMENT_SUB_1",
+        payment_hash="hash_square_subscription",
+        wallet_id="wallet_1",
+        amount=925000,
+        fee=0,
+        bolt11="lnbc1square",
+        fiat_provider="square",
+        extra={
+            "subscription_request_id": "subscription_square_1",
+            "tag": "members",
+            "link": "link-1",
+        },
+        memo="Square Members",
+    )
+    settings.square_api_endpoint = "https://connect.squareupsandbox.com"
+    settings.square_access_token = "square-token"
+    settings.square_location_id = "LOC123"
+    settings.square_api_version = "2026-01-22"
+    square_provider = SquareWallet()
+    mocker.patch.object(
+        square_provider,
+        "get_payment_for_order",
+        return_value={
+            "id": "PAYMENT_SUB_1",
+            "status": "COMPLETED",
+            "amount_money": {"amount": 925, "currency": "USD"},
+        },
+    )
+    mocker.patch(
+        "lnbits.core.views.callback_api.get_fiat_provider",
+        mocker.AsyncMock(return_value=square_provider),
+    )
+    get_standalone_payment_mock = mocker.patch(
+        "lnbits.core.views.callback_api.get_standalone_payment",
+        mocker.AsyncMock(return_value=payment),
+    )
+    update_payment_mock = mocker.patch(
+        "lnbits.core.views.callback_api.update_payment",
+        mocker.AsyncMock(),
+    )
+    get_payments_mock = mocker.patch(
+        "lnbits.core.views.callback_api.get_payments",
+        mocker.AsyncMock(return_value=[]),
+    )
+    create_fiat_invoice_mock = mocker.patch(
+        "lnbits.core.views.callback_api.create_fiat_invoice",
+        mocker.AsyncMock(),
+    )
+    fiat_status_mock = mocker.patch(
+        "lnbits.core.views.callback_api.check_fiat_status", mocker.AsyncMock()
+    )
+
+    await handle_square_event(
+        {
+            "event_id": "evt_square_invoice",
+            "type": "invoice.payment_made",
+            "data": {
+                "object": {
+                    "invoice": {
+                        "order_id": "ORDER_SUB_1",
+                        "subscription_id": "SUBSCRIPTION_1",
+                    }
+                }
+            },
+        }
+    )
+
+    get_standalone_payment_mock.assert_awaited_with("fiat_square_payment_PAYMENT_SUB_1")
+    assert payment.external_id == "SUBSCRIPTION_1"
+    update_payment_mock.assert_awaited_once_with(payment)
+    fiat_status_mock.assert_awaited_once_with(payment)
+    get_payments_mock.assert_not_awaited()
+    create_fiat_invoice_mock.assert_not_awaited()

tests/api/test_payment_api.py

@@ -4,9 +4,10 @@ from uuid import uuid4
 
 import pytest
 from fastapi import HTTPException
+from pydantic import ValidationError
 
-from lnbits.core.crud.payments import create_payment
+from lnbits.core.crud.payments import create_payment, get_payments
-from lnbits.core.models import Account, CreateInvoice, PaymentState
+from lnbits.core.models import Account, CreateInvoice, PaymentFilters, PaymentState
 from lnbits.core.models.payments import CancelInvoice, CreatePayment, SettleInvoice
 from lnbits.core.models.users import AccountId
 from lnbits.core.models.wallets import KeyType, WalletTypeInfo
@@ -21,7 +22,7 @@ from lnbits.core.views.payment_api import (
     api_payments_settle,
     api_payments_wallets_stats,
 )
-from lnbits.db import Filters
+from lnbits.db import Filter, Filters
 from lnbits.wallets.base import InvoiceResponse
 
 ZERO_AMOUNT_INVOICE = (
@@ -91,6 +92,60 @@ async def test_payment_api_stats_and_all_paginated(admin_user):
     assert second_wallet.id in wallet_ids
 
 
+@pytest.mark.anyio
+async def test_payment_external_id_is_stored_and_validated():
+    user = await create_user_account(
+        Account(
+            id=uuid4().hex,
+            username=f"user_{uuid4().hex[:8]}",
+            email=f"user_{uuid4().hex[:8]}@lnbits.com",
+        )
+    )
+    wallet = user.wallets[0]
+
+    first_payment = await create_wallet_invoice(
+        wallet.id,
+        CreateInvoice(
+            out=False,
+            amount=21,
+            memo="external reference",
+            external_id="provider_payment_123",
+        ),
+    )
+    second_payment = await create_wallet_invoice(
+        wallet.id,
+        CreateInvoice(
+            out=False,
+            amount=22,
+            memo="external reference newest",
+            external_id="provider_payment_123",
+        ),
+    )
+
+    assert first_payment.external_id == "provider_payment_123"
+    assert second_payment.external_id == "provider_payment_123"
+    stored_payments = await get_payments(
+        wallet_id=wallet.id,
+        filters=Filters(
+            filters=[
+                Filter.parse_query(
+                    "external_id", ["provider_payment_123"], PaymentFilters
+                )
+            ],
+            model=PaymentFilters,
+            sortby="created_at",
+            direction="desc",
+        ),
+    )
+    assert [payment.checking_id for payment in stored_payments] == [
+        second_payment.checking_id,
+        first_payment.checking_id,
+    ]
+
+    with pytest.raises(ValidationError, match="Invalid external id"):
+        CreateInvoice(out=False, amount=21, external_id="provider payment 123")
+
+
 @pytest.mark.anyio
 async def test_payment_api_fee_reserve_and_hold_invoice_actions(mocker):
     user = await create_user_account(

tests/unit/test_fiat_providers.py

@@ -629,6 +629,41 @@ async def test_square_wallet_cancel_subscription(settings: Settings):
     assert client.calls[0][0] == "/v2/subscriptions/SUBSCRIPTION123/cancel"
 
 
+@pytest.mark.anyio
+async def test_square_wallet_cancel_subscription_by_request_id(
+    settings: Settings, mocker: MockerFixture
+):
+    settings.square_api_endpoint = "https://connect.squareupsandbox.com"
+    settings.square_access_token = "square-token"
+    settings.square_location_id = "LOC123"
+    settings.square_api_version = "2026-01-22"
+
+    wallet = SquareWallet()
+    client = MockHTTPClient([MockHTTPResponse(json_data={"subscription": {}})])
+    wallet.client = client  # type: ignore[assignment]
+    payment = Payment(
+        checking_id="fiat_square_payment_PAYMENT123",
+        payment_hash="hash123",
+        wallet_id="wallet_1",
+        amount=1000,
+        fee=0,
+        bolt11="lnbc1square",
+        fiat_provider="square",
+        extra={"subscription_request_id": "REQUEST123"},
+        external_id="SUBSCRIPTION123",
+    )
+    get_payments_mock = mocker.patch(
+        "lnbits.core.crud.payments.get_payments",
+        AsyncMock(side_effect=[[], [payment]]),
+    )
+
+    response = await wallet.cancel_subscription("REQUEST123", "wallet_1")
+
+    assert response.ok is True
+    assert client.calls[0][0] == "/v2/subscriptions/SUBSCRIPTION123/cancel"
+    assert get_payments_mock.await_count == 2
+
+
 @pytest.mark.anyio
 async def test_square_wallet_get_invoice_status(settings: Settings):
     settings.square_api_endpoint = "https://connect.squareupsandbox.com"