From c77207405b5a1a02986c8ffa9b495d447c3d94c6 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 13 May 2026 12:32:20 +0300 Subject: [PATCH] feat: introduce `external_id` --- lnbits/core/crud/payments.py | 30 +--------- lnbits/core/migrations.py | 13 +++++ lnbits/core/models/payments.py | 28 ++++++++++ lnbits/core/services/payments.py | 5 ++ lnbits/core/views/callback_api.py | 38 ++++++++----- lnbits/core/views/payment_api.py | 1 + lnbits/fiat/square.py | 59 +++++++++++++++++--- tests/api/test_callback_api.py | 91 +++++++++++++++++++++++++++++-- tests/api/test_payment_api.py | 61 ++++++++++++++++++++- tests/unit/test_fiat_providers.py | 35 ++++++++++++ 10 files changed, 303 insertions(+), 58 deletions(-) diff --git a/lnbits/core/crud/payments.py b/lnbits/core/crud/payments.py index f890d8e97..877a32374 100644 --- a/lnbits/core/crud/payments.py +++ b/lnbits/core/crud/payments.py @@ -107,35 +107,6 @@ async def get_latest_payments_by_extension( ) -async def get_latest_payment_by_extra_key_value( - key: str, - value: str, - wallet_id: str | None = None, - conn: Connection | None = None, -) -> Payment | None: - values = { - "extra_key": f'%"{key}"%', - "extra_value": f'%"{value}"%', - } - clause = ["extra LIKE :extra_key", "extra LIKE :extra_value"] - if wallet_id: - wallet = await get_wallet(wallet_id, conn=conn) - if not wallet or not wallet.can_view_payments: - return None - values["wallet_id"] = wallet.source_wallet_id - clause.append("wallet_id = :wallet_id") - - return await (conn or db).fetchone( - f""" - SELECT * FROM apipayments - WHERE {" AND ".join(clause)} - ORDER BY time DESC LIMIT 1 - """, # noqa: S608 - values, - Payment, - ) - - async def get_payments_paginated( # noqa: C901 *, wallet_id: str | None = None, @@ -321,6 +292,7 @@ async def create_payment( tag=extra.get("tag", None), extra=extra, labels=data.labels or [], + external_id=data.external_id, ) await (conn or db).insert("apipayments", payment) diff --git a/lnbits/core/migrations.py b/lnbits/core/migrations.py index e804df160..d39bf5932 100644 --- a/lnbits/core/migrations.py +++ b/lnbits/core/migrations.py @@ -802,3 +802,16 @@ async def m044_add_activated_to_accounts(db: Connection): Used for account activation status. """ await db.execute("ALTER TABLE accounts ADD COLUMN activated BOOLEAN DEFAULT true") + + +async def m045_add_external_id_to_payments(db: Connection): + """ + Adds external_id column to apipayments. + Used for external payment references. + """ + await db.execute("ALTER TABLE apipayments ADD COLUMN external_id TEXT") + logger.debug("Creating index idx_payments_external_id...") + await db.execute(""" + CREATE INDEX IF NOT EXISTS idx_payments_external_id + ON apipayments (external_id); + """) diff --git a/lnbits/core/models/payments.py b/lnbits/core/models/payments.py index b6d530d5f..95abd6f20 100644 --- a/lnbits/core/models/payments.py +++ b/lnbits/core/models/payments.py @@ -13,6 +13,7 @@ from lnbits.db import FilterModel from lnbits.fiat.base import ( FiatPaymentStatus, ) +from lnbits.helpers import is_valid_external_id from lnbits.utils.exchange_rates import allowed_currencies from lnbits.wallets.base import ( PaymentStatus, @@ -53,6 +54,11 @@ class CreatePayment(BaseModel): webhook: str | None = None fee: int = 0 labels: list[str] | None = None + external_id: str | None = None + + @validator("external_id") + def validate_external_id(cls, external_id): + return _validate_external_id(external_id) class Payment(BaseModel): @@ -77,6 +83,11 @@ class Payment(BaseModel): updated_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) labels: list[str] = [] extra: dict = {} + external_id: str | None = None + + @validator("external_id") + def validate_external_id(cls, external_id): + return _validate_external_id(external_id) def __init__(self, **data): super().__init__(**data) @@ -151,6 +162,7 @@ class PaymentFilters(FilterModel): "status", "time", "labels", + "external_id", ] __sort_fields__ = [ @@ -161,11 +173,13 @@ class PaymentFilters(FilterModel): "memo", "time", "tag", + "external_id", ] status: str | None tag: str | None checking_id: str | None + external_id: str | None amount: int fee: int memo: str | None @@ -249,6 +263,7 @@ class CreateInvoice(BaseModel): lnurl_withdraw: LnurlWithdrawResponse | None = None fiat_provider: str | None = None labels: list[str] = [] + external_id: str | None = Query(default=None, max_length=256) @validator("payment_hash") def check_hex(cls, v): @@ -263,6 +278,10 @@ class CreateInvoice(BaseModel): raise ValueError("The provided unit is not supported") return v + @validator("external_id") + def validate_external_id(cls, external_id): + return _validate_external_id(external_id) + class PaymentsStatusCount(BaseModel): incoming: int = 0 @@ -301,3 +320,12 @@ class CancelInvoice(BaseModel): class UpdatePaymentLabels(BaseModel): labels: list[str] = [] + + +def _validate_external_id(external_id: str | None) -> str | None: + if external_id and not is_valid_external_id(external_id): + raise ValueError( + "Invalid external id. Max length is 256 characters. " + "Space and newlines are not allowed." + ) + return external_id diff --git a/lnbits/core/services/payments.py b/lnbits/core/services/payments.py index a3203d467..a0f241701 100644 --- a/lnbits/core/services/payments.py +++ b/lnbits/core/services/payments.py @@ -64,6 +64,7 @@ async def pay_invoice( description: str = "", tag: str = "", labels: list[str] | None = None, + external_id: str | None = None, conn: Connection | None = None, ) -> Payment: if settings.lnbits_only_allow_incoming_payments: @@ -97,6 +98,7 @@ async def pay_invoice( memo=description or invoice.description or "", extra=extra, labels=labels, + external_id=external_id, ) async with db.reuse_conn(conn) if conn else db.connect() as new_conn: @@ -217,6 +219,7 @@ async def create_wallet_invoice(wallet_id: str, data: CreateInvoice) -> Payment: internal=data.internal, payment_hash=data.payment_hash, labels=data.labels, + external_id=data.external_id, conn=conn, ) @@ -258,6 +261,7 @@ async def create_invoice( internal: bool | None = False, payment_hash: str | None = None, labels: list[str] | None = None, + external_id: str | None = None, conn: Connection | None = None, ) -> Payment: if not amount > 0: @@ -342,6 +346,7 @@ async def create_invoice( webhook=webhook, fee=invoice_response.fee_msat or 0, labels=labels, + external_id=external_id, ) payment = await create_payment( diff --git a/lnbits/core/views/callback_api.py b/lnbits/core/views/callback_api.py index f12c29a95..6ee9b350e 100644 --- a/lnbits/core/views/callback_api.py +++ b/lnbits/core/views/callback_api.py @@ -4,11 +4,11 @@ from fastapi import APIRouter, Request from loguru import logger from lnbits.core.crud.payments import ( - get_latest_payment_by_extra_key_value, + get_payments, get_standalone_payment, update_payment, ) -from lnbits.core.models import Payment +from lnbits.core.models import Payment, PaymentFilters from lnbits.core.models.misc import SimpleStatus from lnbits.core.models.payments import CreateInvoice from lnbits.core.services.fiat_providers import ( @@ -18,6 +18,7 @@ from lnbits.core.services.fiat_providers import ( verify_paypal_webhook, ) from lnbits.core.services.payments import create_fiat_invoice +from lnbits.db import Filter, Filters from lnbits.fiat import get_fiat_provider from lnbits.fiat.base import FiatSubscriptionPaymentOptions from lnbits.fiat.square import SquareWallet @@ -364,13 +365,27 @@ async def _handle_square_invoice_payment_made(event: dict): payment_options = _deserialize_square_metadata(_square_payment_note(payment)) if not payment_options.wallet_id: + payment_id = payment.get("id") stored_payment = ( - await get_latest_payment_by_extra_key_value( - "square_subscription_id", subscription_id - ) - if subscription_id + await get_standalone_payment(f"fiat_square_payment_{payment_id}") + if payment_id else None ) + if not stored_payment and subscription_id: + stored_payments = await get_payments( + filters=Filters( + filters=[ + Filter.parse_query( + "external_id", [subscription_id], PaymentFilters + ) + ], + model=PaymentFilters, + sortby="created_at", + direction="desc", + limit=1, + ) + ) + stored_payment = stored_payments[0] if stored_payments else None if stored_payment: payment_options = _square_payment_options_from_payment(stored_payment) else: @@ -406,10 +421,9 @@ async def _handle_square_subscription_payment( if existing_payment: if ( square_subscription_id - and (existing_payment.extra or {}).get("square_subscription_id") - != square_subscription_id + and existing_payment.external_id != square_subscription_id ): - existing_payment.extra["square_subscription_id"] = square_subscription_id + existing_payment.external_id = square_subscription_id await update_payment(existing_payment) await check_fiat_status(existing_payment) return @@ -427,8 +441,6 @@ async def _handle_square_subscription_payment( "payment_request": payment_request, }, } - if square_subscription_id: - extra["square_subscription_id"] = square_subscription_id lnbits_payment = await create_fiat_invoice( wallet_id=wallet_id, @@ -438,6 +450,7 @@ async def _handle_square_subscription_payment( memo=payment_options.memo or "", extra=extra, fiat_provider="square", + external_id=square_subscription_id, ), ) @@ -493,6 +506,5 @@ def _deserialize_square_metadata(custom_id: str) -> FiatSubscriptionPaymentOptio extra=extra, memo=memo, ) - except (json.JSONDecodeError, IndexError, TypeError) as e: - logger.debug(f"Failed to deserialize Square metadata: {e}") + except (json.JSONDecodeError, IndexError, TypeError): return FiatSubscriptionPaymentOptions() diff --git a/lnbits/core/views/payment_api.py b/lnbits/core/views/payment_api.py index 38d6e10df..b8509c0f1 100644 --- a/lnbits/core/views/payment_api.py +++ b/lnbits/core/views/payment_api.py @@ -263,6 +263,7 @@ async def api_payments_create( payment_request=invoice_data.bolt11, extra=invoice_data.extra, labels=invoice_data.labels, + external_id=invoice_data.external_id, ) return payment diff --git a/lnbits/fiat/square.py b/lnbits/fiat/square.py index 1c055c712..a68c7f4c7 100644 --- a/lnbits/fiat/square.py +++ b/lnbits/fiat/square.py @@ -312,6 +312,7 @@ class SquareWallet(FiatProvider): ) -> SquareSubscriptionCheckoutInfo: plan_data = catalog_object.get("subscription_plan_data") or {} plan_variations = plan_data.get("subscription_plan_variations") or [] + eligible_item_ids = plan_data.get("eligible_item_ids") or [] plan_variation = next( ( variation @@ -325,7 +326,7 @@ class SquareWallet(FiatProvider): price_money = await self._get_subscription_price_money( plan_variation, - eligible_item_ids=plan_data.get("eligible_item_ids") or [], + eligible_item_ids=eligible_item_ids, ) plan_variation_id = plan_variation.get("id") if not plan_variation_id: @@ -547,15 +548,57 @@ class SquareWallet(FiatProvider): async def _get_square_subscription_id( self, subscription_id: str, wallet_id: str ) -> str: - from lnbits.core.crud.payments import get_latest_payment_by_extra_key_value + from lnbits.core.crud.payments import get_payments + from lnbits.core.models import PaymentFilters + from lnbits.db import Filter, Filters - payment = await get_latest_payment_by_extra_key_value( - "subscription_request_id", subscription_id, wallet_id=wallet_id + payments = await get_payments( + wallet_id=wallet_id, + filters=Filters( + filters=[ + Filter.parse_query("external_id", [subscription_id], PaymentFilters) + ], + model=PaymentFilters, + sortby="created_at", + direction="desc", + limit=1, + ), ) - if not payment: - return subscription_id - square_subscription_id = (payment.extra or {}).get("square_subscription_id") - return square_subscription_id or subscription_id + payment = next( + ( + payment + for payment in payments + if payment.external_id and payment.fiat_provider == "square" + ), + None, + ) + if payment and payment.external_id: + return payment.external_id + + payments = await get_payments( + wallet_id=wallet_id, + incoming=True, + filters=Filters( + model=PaymentFilters, + sortby="created_at", + direction="desc", + ), + ) + payment = next( + ( + payment + for payment in payments + if payment.external_id + and payment.fiat_provider == "square" + and (payment.extra or {}).get("subscription_request_id") + == subscription_id + ), + None, + ) + if payment and payment.external_id: + return payment.external_id + + return subscription_id def _settings_connection_fields(self) -> str: return "-".join( diff --git a/tests/api/test_callback_api.py b/tests/api/test_callback_api.py index 06d8b7f79..bd6b9e6e0 100644 --- a/tests/api/test_callback_api.py +++ b/tests/api/test_callback_api.py @@ -4,7 +4,7 @@ from uuid import uuid4 import pytest from httpx import AsyncClient -from lnbits.core.models import Account, CreateInvoice +from lnbits.core.models import Account, CreateInvoice, Payment from lnbits.core.services.payments import create_wallet_invoice from lnbits.core.services.users import create_user_account from lnbits.core.views.callback_api import ( @@ -253,9 +253,9 @@ async def test_callback_api_handles_subscription_flows_and_validation( payment.extra = { "subscription_request_id": "subscription_square_1", "tag": "members", - "square_subscription_id": "SUBSCRIPTION_1", "link": "link-1", } + payment.external_id = "SUBSCRIPTION_1" payment.memo = "Square Members" settings.square_api_endpoint = "https://connect.squareupsandbox.com" settings.square_access_token = "square-token" @@ -276,8 +276,8 @@ async def test_callback_api_handles_subscription_flows_and_validation( mocker.AsyncMock(return_value=square_provider), ) mocker.patch( - "lnbits.core.views.callback_api.get_latest_payment_by_extra_key_value", - mocker.AsyncMock(return_value=payment), + "lnbits.core.views.callback_api.get_payments", + mocker.AsyncMock(return_value=[payment]), ) await handle_square_event( @@ -298,7 +298,8 @@ async def test_callback_api_handles_subscription_flows_and_validation( assert create_fiat_invoice_mock.await_count == 4 square_invoice_call = create_fiat_invoice_mock.await_args.kwargs square_invoice = square_invoice_call["invoice_data"] - assert square_invoice.extra["square_subscription_id"] == "SUBSCRIPTION_1" + assert square_invoice.external_id == "SUBSCRIPTION_1" + assert "square_subscription_id" not in square_invoice.extra assert ( square_invoice.extra["subscription"]["payment_request"] == "https://square.example/invoice" @@ -314,3 +315,83 @@ async def test_callback_api_handles_subscription_flows_and_validation( "resource": {"amount": {"currency": "USD", "total": "5.00"}}, } ) + + +@pytest.mark.anyio +async def test_square_invoice_payment_updates_existing_subscription_external_id( + settings: Settings, mocker +): + payment = Payment( + checking_id="fiat_square_payment_PAYMENT_SUB_1", + payment_hash="hash_square_subscription", + wallet_id="wallet_1", + amount=925000, + fee=0, + bolt11="lnbc1square", + fiat_provider="square", + extra={ + "subscription_request_id": "subscription_square_1", + "tag": "members", + "link": "link-1", + }, + memo="Square Members", + ) + settings.square_api_endpoint = "https://connect.squareupsandbox.com" + settings.square_access_token = "square-token" + settings.square_location_id = "LOC123" + settings.square_api_version = "2026-01-22" + square_provider = SquareWallet() + mocker.patch.object( + square_provider, + "get_payment_for_order", + return_value={ + "id": "PAYMENT_SUB_1", + "status": "COMPLETED", + "amount_money": {"amount": 925, "currency": "USD"}, + }, + ) + mocker.patch( + "lnbits.core.views.callback_api.get_fiat_provider", + mocker.AsyncMock(return_value=square_provider), + ) + get_standalone_payment_mock = mocker.patch( + "lnbits.core.views.callback_api.get_standalone_payment", + mocker.AsyncMock(return_value=payment), + ) + update_payment_mock = mocker.patch( + "lnbits.core.views.callback_api.update_payment", + mocker.AsyncMock(), + ) + get_payments_mock = mocker.patch( + "lnbits.core.views.callback_api.get_payments", + mocker.AsyncMock(return_value=[]), + ) + create_fiat_invoice_mock = mocker.patch( + "lnbits.core.views.callback_api.create_fiat_invoice", + mocker.AsyncMock(), + ) + fiat_status_mock = mocker.patch( + "lnbits.core.views.callback_api.check_fiat_status", mocker.AsyncMock() + ) + + await handle_square_event( + { + "event_id": "evt_square_invoice", + "type": "invoice.payment_made", + "data": { + "object": { + "invoice": { + "order_id": "ORDER_SUB_1", + "subscription_id": "SUBSCRIPTION_1", + } + } + }, + } + ) + + get_standalone_payment_mock.assert_awaited_with("fiat_square_payment_PAYMENT_SUB_1") + assert payment.external_id == "SUBSCRIPTION_1" + update_payment_mock.assert_awaited_once_with(payment) + fiat_status_mock.assert_awaited_once_with(payment) + get_payments_mock.assert_not_awaited() + create_fiat_invoice_mock.assert_not_awaited() diff --git a/tests/api/test_payment_api.py b/tests/api/test_payment_api.py index 1371b47cc..76c60bafd 100644 --- a/tests/api/test_payment_api.py +++ b/tests/api/test_payment_api.py @@ -4,9 +4,10 @@ from uuid import uuid4 import pytest from fastapi import HTTPException +from pydantic import ValidationError -from lnbits.core.crud.payments import create_payment -from lnbits.core.models import Account, CreateInvoice, PaymentState +from lnbits.core.crud.payments import create_payment, get_payments +from lnbits.core.models import Account, CreateInvoice, PaymentFilters, PaymentState from lnbits.core.models.payments import CancelInvoice, CreatePayment, SettleInvoice from lnbits.core.models.users import AccountId from lnbits.core.models.wallets import KeyType, WalletTypeInfo @@ -21,7 +22,7 @@ from lnbits.core.views.payment_api import ( api_payments_settle, api_payments_wallets_stats, ) -from lnbits.db import Filters +from lnbits.db import Filter, Filters from lnbits.wallets.base import InvoiceResponse ZERO_AMOUNT_INVOICE = ( @@ -91,6 +92,60 @@ async def test_payment_api_stats_and_all_paginated(admin_user): assert second_wallet.id in wallet_ids +@pytest.mark.anyio +async def test_payment_external_id_is_stored_and_validated(): + user = await create_user_account( + Account( + id=uuid4().hex, + username=f"user_{uuid4().hex[:8]}", + email=f"user_{uuid4().hex[:8]}@lnbits.com", + ) + ) + wallet = user.wallets[0] + + first_payment = await create_wallet_invoice( + wallet.id, + CreateInvoice( + out=False, + amount=21, + memo="external reference", + external_id="provider_payment_123", + ), + ) + second_payment = await create_wallet_invoice( + wallet.id, + CreateInvoice( + out=False, + amount=22, + memo="external reference newest", + external_id="provider_payment_123", + ), + ) + + assert first_payment.external_id == "provider_payment_123" + assert second_payment.external_id == "provider_payment_123" + stored_payments = await get_payments( + wallet_id=wallet.id, + filters=Filters( + filters=[ + Filter.parse_query( + "external_id", ["provider_payment_123"], PaymentFilters + ) + ], + model=PaymentFilters, + sortby="created_at", + direction="desc", + ), + ) + assert [payment.checking_id for payment in stored_payments] == [ + second_payment.checking_id, + first_payment.checking_id, + ] + + with pytest.raises(ValidationError, match="Invalid external id"): + CreateInvoice(out=False, amount=21, external_id="provider payment 123") + + @pytest.mark.anyio async def test_payment_api_fee_reserve_and_hold_invoice_actions(mocker): user = await create_user_account( diff --git a/tests/unit/test_fiat_providers.py b/tests/unit/test_fiat_providers.py index 6553159da..7b6f1a6e0 100644 --- a/tests/unit/test_fiat_providers.py +++ b/tests/unit/test_fiat_providers.py @@ -629,6 +629,41 @@ async def test_square_wallet_cancel_subscription(settings: Settings): assert client.calls[0][0] == "/v2/subscriptions/SUBSCRIPTION123/cancel" +@pytest.mark.anyio +async def test_square_wallet_cancel_subscription_by_request_id( + settings: Settings, mocker: MockerFixture +): + settings.square_api_endpoint = "https://connect.squareupsandbox.com" + settings.square_access_token = "square-token" + settings.square_location_id = "LOC123" + settings.square_api_version = "2026-01-22" + + wallet = SquareWallet() + client = MockHTTPClient([MockHTTPResponse(json_data={"subscription": {}})]) + wallet.client = client # type: ignore[assignment] + payment = Payment( + checking_id="fiat_square_payment_PAYMENT123", + payment_hash="hash123", + wallet_id="wallet_1", + amount=1000, + fee=0, + bolt11="lnbc1square", + fiat_provider="square", + extra={"subscription_request_id": "REQUEST123"}, + external_id="SUBSCRIPTION123", + ) + get_payments_mock = mocker.patch( + "lnbits.core.crud.payments.get_payments", + AsyncMock(side_effect=[[], [payment]]), + ) + + response = await wallet.cancel_subscription("REQUEST123", "wallet_1") + + assert response.ok is True + assert client.calls[0][0] == "/v2/subscriptions/SUBSCRIPTION123/cancel" + assert get_payments_mock.await_count == 2 + + @pytest.mark.anyio async def test_square_wallet_get_invoice_status(settings: Settings): settings.square_api_endpoint = "https://connect.squareupsandbox.com"