fix: pay invoice status (#2481)

* fix: rest `pay_invoice` pending instead of failed
* fix: rpc `pay_invoice` pending instead of failed
* fix: return "failed" value for payment
* fix: handle failed status for LNbits funding source
* chore: `phoenixd` todo
* test: fix condition
* fix: wait for payment status to be updated
* fix: fail payment when explicit status provided

---------

Co-authored-by: dni ⚡ <office@dnilabs.com>

lnbits/core/services.py

@@ -63,11 +63,15 @@ from .models import Payment, UserConfig, Wallet
 
 
 class PaymentError(Exception):
-    pass
+    def __init__(self, message: str, status: str = "pending"):
+        self.message = message
+        self.status = status
 
 
 class InvoiceError(Exception):
-    pass
+    def __init__(self, message: str, status: str = "pending"):
+        self.message = message
+        self.status = status
 
 
 async def calculate_fiat_amounts(
@@ -123,11 +127,11 @@ async def create_invoice(
     conn: Optional[Connection] = None,
 ) -> Tuple[str, str]:
     if not amount > 0:
-        raise InvoiceError("Amountless invoices not supported.")
+        raise InvoiceError("Amountless invoices not supported.", status="failed")
 
     user_wallet = await get_wallet(wallet_id, conn=conn)
     if not user_wallet:
-        raise InvoiceError(f"Could not fetch wallet '{wallet_id}'.")
+        raise InvoiceError(f"Could not fetch wallet '{wallet_id}'.", status="failed")
 
     invoice_memo = None if description_hash else memo
 
@@ -142,8 +146,9 @@ async def create_invoice(
         user_wallet.balance_msat / 1000 + amount_sat
     ):
         raise InvoiceError(
-            f"Wallet balance  cannot exceed "
-            f"{settings.lnbits_wallet_limit_max_balance} sats."
+            f"Wallet balance cannot exceed "
+            f"{settings.lnbits_wallet_limit_max_balance} sats.",
+            status="failed",
         )
 
     (
@@ -159,7 +164,9 @@ async def create_invoice(
         expiry=expiry or settings.lightning_invoice_expiry,
     )
     if not ok or not payment_request or not checking_id:
-        raise InvoiceError(error_message or "unexpected backend error.")
+        raise InvoiceError(
+            error_message or "unexpected backend error.", status="pending"
+        )
 
     invoice = bolt11_decode(payment_request)
 
@@ -202,12 +209,12 @@ async def pay_invoice(
     try:
         invoice = bolt11_decode(payment_request)
     except Exception as exc:
-        raise InvoiceError("Bolt11 decoding failed.") from exc
+        raise PaymentError("Bolt11 decoding failed.", status="failed") from exc
 
     if not invoice.amount_msat or not invoice.amount_msat > 0:
-        raise InvoiceError("Amountless invoices not supported.")
+        raise PaymentError("Amountless invoices not supported.", status="failed")
     if max_sat and invoice.amount_msat > max_sat * 1000:
-        raise InvoiceError("Amount in invoice is too high.")
+        raise PaymentError("Amount in invoice is too high.", status="failed")
 
     await check_wallet_limits(wallet_id, conn, invoice.amount_msat)
 
@@ -242,7 +249,7 @@ async def pay_invoice(
         # we check if an internal invoice exists that has already been paid
         # (not pending anymore)
         if not await check_internal_pending(invoice.payment_hash, conn=conn):
-            raise PaymentError("Internal invoice already paid.")
+            raise PaymentError("Internal invoice already paid.", status="failed")
 
         # check_internal() returns the checking_id of the invoice we're waiting for
         # (pending only)
@@ -261,7 +268,7 @@ async def pay_invoice(
                 internal_invoice.amount != invoice.amount_msat
                 or internal_invoice.bolt11 != payment_request.lower()
             ):
-                raise PaymentError("Invalid invoice.")
+                raise PaymentError("Invalid invoice.", status="failed")
 
             logger.debug(f"creating temporary internal payment with id {internal_id}")
             # create a new payment from this wallet
@@ -289,7 +296,7 @@ async def pay_invoice(
             except Exception as exc:
                 logger.error(f"could not create temporary payment: {exc}")
                 # happens if the same wallet tries to pay an invoice twice
-                raise PaymentError("Could not make payment.") from exc
+                raise PaymentError("Could not make payment.", status="failed") from exc
 
         # do the balance check
         wallet = await get_wallet(wallet_id, conn=conn)
@@ -302,9 +309,10 @@ async def pay_invoice(
             ):
                 raise PaymentError(
                     f"You must reserve at least ({round(fee_reserve_total_msat/1000)}"
-                    "  sat) to cover potential routing fees."
+                    "  sat) to cover potential routing fees.",
+                    status="failed",
                 )
-            raise PermissionError("Insufficient balance.")
+            raise PaymentError("Insufficient balance.", status="failed")
 
     if internal_checking_id:
         service_fee_msat = service_fee(invoice.amount_msat, internal=True)
@@ -340,6 +348,7 @@ async def pay_invoice(
             )
 
         logger.debug(f"backend: pay_invoice finished {temp_id}")
+        logger.debug(f"backend: pay_invoice response {payment}")
         if payment.checking_id and payment.ok is not False:
             # payment.ok can be True (paid) or None (pending)!
             logger.debug(f"updating payment {temp_id}")
@@ -370,7 +379,8 @@ async def pay_invoice(
                 await delete_wallet_payment(temp_id, wallet_id, conn=conn)
             raise PaymentError(
                 f"Payment failed: {payment.error_message}"
-                or "Payment failed, but backend didn't give us an error message."
+                or "Payment failed, but backend didn't give us an error message.",
+                status="failed",
             )
         else:
             logger.warning(
@@ -413,8 +423,9 @@ async def check_time_limit_between_transactions(conn, wallet_id):
     if len(payments) == 0:
         return
 
-    raise ValueError(
-        f"The time limit of {limit} seconds between payments has been reached."
+    raise PaymentError(
+        status="failed",
+        message=f"The time limit of {limit} seconds between payments has been reached.",
     )
 
 

lnbits/core/views/payment_api.py

@@ -171,7 +171,10 @@ async def api_payments_create_invoice(data: CreateInvoice, wallet: Wallet):
             assert payment_db is not None, "payment not found"
             checking_id = payment_db.checking_id
         except InvoiceError as exc:
-            raise HTTPException(status_code=520, detail=str(exc)) from exc
+            return JSONResponse(
+                status_code=520,
+                content={"detail": exc.message, "status": exc.status},
+            )
         except Exception as exc:
             raise exc
 
@@ -217,14 +220,11 @@ async def api_payments_pay_invoice(
         payment_hash = await pay_invoice(
             wallet_id=wallet.id, payment_request=bolt11, extra=extra
         )
-    except ValueError as exc:
-        raise HTTPException(
-            status_code=HTTPStatus.BAD_REQUEST, detail=str(exc)
-        ) from exc
-    except PermissionError as exc:
-        raise HTTPException(status_code=HTTPStatus.FORBIDDEN, detail=str(exc)) from exc
     except PaymentError as exc:
-        raise HTTPException(status_code=520, detail=str(exc)) from exc
+        return JSONResponse(
+            status_code=520,
+            content={"detail": exc.message, "status": exc.status},
+        )
     except Exception as exc:
         raise exc
 
@@ -434,7 +434,7 @@ async def api_payment(payment_hash, x_api_key: Optional[str] = Header(None)):
         return {"paid": True, "preimage": payment.preimage}
 
     try:
-        await payment.check_status()
+        status = await payment.check_status()
     except Exception:
         if wallet and wallet.id == payment.wallet_id:
             return {"paid": False, "details": payment}
@@ -443,6 +443,7 @@ async def api_payment(payment_hash, x_api_key: Optional[str] = Header(None)):
     if wallet and wallet.id == payment.wallet_id:
         return {
             "paid": not payment.pending,
+            "status": f"{status!s}",
             "preimage": payment.preimage,
             "details": payment,
         }

lnbits/wallets/alby.py

@@ -129,7 +129,7 @@ class AlbyWallet(Wallet):
 
             if r.is_error:
                 error_message = data["message"] if "message" in data else r.text
-                return PaymentResponse(False, None, None, None, error_message)
+                return PaymentResponse(None, None, None, None, error_message)
 
             checking_id = data["payment_hash"]
             # todo: confirm with bitkarrot that having the minus is fine
@@ -141,18 +141,18 @@ class AlbyWallet(Wallet):
         except KeyError as exc:
             logger.warning(exc)
             return PaymentResponse(
-                False, None, None, None, "Server error: 'missing required fields'"
+                None, None, None, None, "Server error: 'missing required fields'"
             )
         except json.JSONDecodeError as exc:
             logger.warning(exc)
             return PaymentResponse(
-                False, None, None, None, "Server error: 'invalid json response'"
+                None, None, None, None, "Server error: 'invalid json response'"
             )
         except Exception as exc:
             logger.info(f"Failed to pay invoice {bolt11}")
             logger.warning(exc)
             return PaymentResponse(
-                False, None, None, None, f"Unable to connect to {self.endpoint}."
+                None, None, None, None, f"Unable to connect to {self.endpoint}."
             )
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
@@ -167,6 +167,7 @@ class AlbyWallet(Wallet):
 
             data = r.json()
 
+            # TODO: how can we detect a failed payment?
             statuses = {
                 "CREATED": None,
                 "SETTLED": True,

lnbits/wallets/base.py

@@ -70,14 +70,11 @@ class PaymentStatus(NamedTuple):
         return self.paid is False
 
     def __str__(self) -> str:
-        if self.paid is True:
-            return "settled"
-        elif self.paid is False:
+        if self.success:
+            return "success"
+        if self.failed:
             return "failed"
-        elif self.paid is None:
-            return "still pending"
-        else:
-            return "unknown (should never happen)"
+        return "pending"
 
 
 class PaymentSuccessStatus(PaymentStatus):

lnbits/wallets/corelightning.py

@@ -46,6 +46,15 @@ class CoreLightningWallet(Wallet):
         command = self.ln.help("invoice")["help"][0]["command"]  # type: ignore
         self.supports_description_hash = "deschashonly" in command
 
+        # https://docs.corelightning.org/reference/lightning-pay
+        # 201: Already paid
+        # 203: Permanent failure at destination.
+        # 205: Unable to find a route.
+        # 206: Route too expensive.
+        # 207: Invoice expired.
+        # 210: Payment timed out without a payment in progress.
+        self.pay_failure_error_codes = [201, 203, 205, 206, 207, 210]
+
         # check last payindex so we can listen from that point on
         self.last_pay_index = 0
         invoices: dict = self.ln.listinvoices()  # type: ignore
@@ -155,19 +164,27 @@ class CoreLightningWallet(Wallet):
         except RpcError as exc:
             logger.warning(exc)
             try:
-                error_message = exc.error["attempts"][-1]["fail_reason"]  # type: ignore
+                error_code = exc.error.get("code")
+                if error_code in self.pay_failure_error_codes:  # type: ignore
+                    error_message = exc.error.get("message", error_code)  # type: ignore
+                    return PaymentResponse(
+                        False, None, None, None, f"Payment failed: {error_message}"
+                    )
+                else:
+                    error_message = f"Payment failed: {exc.error}"
+                    return PaymentResponse(None, None, None, None, error_message)
             except Exception:
                 error_message = f"RPC '{exc.method}' failed with '{exc.error}'."
-            return PaymentResponse(False, None, None, None, error_message)
+                return PaymentResponse(None, None, None, None, error_message)
         except KeyError as exc:
             logger.warning(exc)
             return PaymentResponse(
-                False, None, None, None, "Server error: 'missing required fields'"
+                None, None, None, None, "Server error: 'missing required fields'"
             )
         except Exception as exc:
             logger.info(f"Failed to pay invoice {bolt11}")
             logger.warning(exc)
-            return PaymentResponse(False, None, None, None, f"Payment failed: '{exc}'.")
+            return PaymentResponse(None, None, None, None, f"Payment failed: '{exc}'.")
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
         try:

lnbits/wallets/corelightningrest.py

@@ -49,6 +49,15 @@ class CoreLightningRestWallet(Wallet):
             "User-Agent": settings.user_agent,
         }
 
+        # https://docs.corelightning.org/reference/lightning-pay
+        # 201: Already paid
+        # 203: Permanent failure at destination.
+        # 205: Unable to find a route.
+        # 206: Route too expensive.
+        # 207: Invoice expired.
+        # 210: Payment timed out without a payment in progress.
+        self.pay_failure_error_codes = [201, 203, 205, 206, 207, 210]
+
         self.cert = settings.corelightning_rest_cert or False
         self.client = httpx.AsyncClient(verify=self.cert, headers=headers)
         self.last_pay_index = 0
@@ -176,37 +185,48 @@ class CoreLightningRestWallet(Wallet):
             r.raise_for_status()
             data = r.json()
 
-            if "error" in data:
-                return PaymentResponse(False, None, None, None, data["error"])
-            if r.is_error:
-                return PaymentResponse(False, None, None, None, r.text)
-            if (
-                "payment_hash" not in data
-                or "payment_preimage" not in data
-                or "msatoshi_sent" not in data
-                or "msatoshi" not in data
-                or "status" not in data
-            ):
+            status = self.statuses.get(data["status"])
+            if "payment_preimage" not in data:
                 return PaymentResponse(
-                    False, None, None, None, "Server error: 'missing required fields'"
+                    status,
+                    None,
+                    None,
+                    None,
+                    data.get("error"),
                 )
 
             checking_id = data["payment_hash"]
             preimage = data["payment_preimage"]
             fee_msat = data["msatoshi_sent"] - data["msatoshi"]
 
-            return PaymentResponse(
-                self.statuses.get(data["status"]), checking_id, fee_msat, preimage, None
-            )
+            return PaymentResponse(status, checking_id, fee_msat, preimage, None)
+        except httpx.HTTPStatusError as exc:
+            try:
+                logger.debug(exc)
+                data = exc.response.json()
+                if data["error"]["code"] in self.pay_failure_error_codes:  # type: ignore
+                    error_message = f"Payment failed: {data['error']['message']}"
+                    return PaymentResponse(False, None, None, None, error_message)
+                error_message = f"REST failed with {data['error']['message']}."
+                return PaymentResponse(None, None, None, None, error_message)
+            except Exception as exc:
+                error_message = f"Unable to connect to {self.url}."
+                return PaymentResponse(None, None, None, None, error_message)
+
         except json.JSONDecodeError:
             return PaymentResponse(
-                False, None, None, None, "Server error: 'invalid json response'"
+                None, None, None, None, "Server error: 'invalid json response'"
+            )
+        except KeyError as exc:
+            logger.warning(exc)
+            return PaymentResponse(
+                None, None, None, None, "Server error: 'missing required fields'"
             )
         except Exception as exc:
             logger.info(f"Failed to pay invoice {bolt11}")
             logger.warning(exc)
             return PaymentResponse(
-                False, None, None, None, f"Unable to connect to {self.url}."
+                None, None, None, None, f"Unable to connect to {self.url}."
             )
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:

lnbits/wallets/eclair.py

@@ -142,9 +142,9 @@ class EclairWallet(Wallet):
             data = r.json()
 
             if "error" in data:
-                return PaymentResponse(False, None, None, None, data["error"])
+                return PaymentResponse(None, None, None, None, data["error"])
             if r.is_error:
-                return PaymentResponse(False, None, None, None, r.text)
+                return PaymentResponse(None, None, None, None, r.text)
 
             if data["type"] == "payment-failed":
                 return PaymentResponse(False, None, None, None, "payment failed")
@@ -154,17 +154,17 @@ class EclairWallet(Wallet):
 
         except json.JSONDecodeError:
             return PaymentResponse(
-                False, None, None, None, "Server error: 'invalid json response'"
+                None, None, None, None, "Server error: 'invalid json response'"
             )
         except KeyError:
             return PaymentResponse(
-                False, None, None, None, "Server error: 'missing required fields'"
+                None, None, None, None, "Server error: 'missing required fields'"
             )
         except Exception as exc:
             logger.info(f"Failed to pay invoice {bolt11}")
             logger.warning(exc)
             return PaymentResponse(
-                False, None, None, None, f"Unable to connect to {self.url}."
+                None, None, None, None, f"Unable to connect to {self.url}."
             )
 
         payment_status: PaymentStatus = await self.get_payment_status(checking_id)

lnbits/wallets/lnbits.py

@@ -9,6 +9,7 @@ from lnbits.settings import settings
 
 from .base import (
     InvoiceResponse,
+    PaymentFailedStatus,
     PaymentPendingStatus,
     PaymentResponse,
     PaymentStatus,
@@ -115,13 +116,10 @@ class LNbitsWallet(Wallet):
                 json={"out": True, "bolt11": bolt11},
                 timeout=None,
             )
+
             r.raise_for_status()
             data = r.json()
 
-            if r.is_error or "payment_hash" not in data:
-                error_message = data["detail"] if "detail" in data else r.text
-                return PaymentResponse(False, None, None, None, error_message)
-
             checking_id = data["payment_hash"]
 
             # we do this to get the fee and preimage
@@ -131,19 +129,32 @@ class LNbitsWallet(Wallet):
             return PaymentResponse(
                 success, checking_id, payment.fee_msat, payment.preimage
             )
+
+        except httpx.HTTPStatusError as exc:
+            try:
+                logger.debug(exc)
+                data = exc.response.json()
+                error_message = f"Payment {data['status']}: {data['detail']}."
+                if data["status"] == "failed":
+                    return PaymentResponse(False, None, None, None, error_message)
+                return PaymentResponse(None, None, None, None, error_message)
+            except Exception as exc:
+                error_message = f"Unable to connect to {self.endpoint}."
+                return PaymentResponse(None, None, None, None, error_message)
+
         except json.JSONDecodeError:
             return PaymentResponse(
-                False, None, None, None, "Server error: 'invalid json response'"
+                None, None, None, None, "Server error: 'invalid json response'"
             )
         except KeyError:
             return PaymentResponse(
-                False, None, None, None, "Server error: 'missing required fields'"
+                None, None, None, None, "Server error: 'missing required fields'"
             )
         except Exception as exc:
             logger.info(f"Failed to pay invoice {bolt11}")
             logger.warning(exc)
             return PaymentResponse(
-                False, None, None, None, f"Unable to connect to {self.endpoint}."
+                None, None, None, None, f"Unable to connect to {self.endpoint}."
             )
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
@@ -169,6 +180,9 @@ class LNbitsWallet(Wallet):
                 return PaymentPendingStatus()
             data = r.json()
 
+            if data.get("status") == "failed":
+                return PaymentFailedStatus()
+
             if "paid" not in data or not data["paid"]:
                 return PaymentPendingStatus()
 

lnbits/wallets/lndgrpc.py

@@ -167,7 +167,7 @@ class LndWallet(Wallet):
             resp = await self.routerpc.SendPaymentV2(req).read()
         except Exception as exc:
             logger.warning(exc)
-            return PaymentResponse(False, None, None, None, str(exc))
+            return PaymentResponse(None, None, None, None, str(exc))
 
         # PaymentStatus from https://github.com/lightningnetwork/lnd/blob/master/channeldb/payments.go#L178
         statuses = {

lnbits/wallets/lndrest.py

@@ -174,39 +174,30 @@ class LndRestWallet(Wallet):
                 timeout=None,
             )
             r.raise_for_status()
-        except Exception as exc:
-            logger.warning(f"LndRestWallet pay_invoice POST error: {exc}.")
-            return PaymentResponse(
-                False, None, None, None, f"Unable to connect to {self.endpoint}."
-            )
-
-        try:
             data = r.json()
 
-            if data.get("payment_error"):
-                error_message = r.json().get("payment_error") or r.text
-                logger.warning(
-                    f"LndRestWallet pay_invoice payment_error: {error_message}."
-                )
-                return PaymentResponse(False, None, None, None, error_message)
-
-            if (
-                "payment_hash" not in data
-                or "payment_route" not in data
-                or "total_fees_msat" not in data["payment_route"]
-                or "payment_preimage" not in data
-            ):
-                return PaymentResponse(
-                    False, None, None, None, "Server error: 'missing required fields'"
-                )
+            payment_error = data.get("payment_error")
+            if payment_error:
+                logger.warning(f"LndRestWallet payment_error: {payment_error}.")
+                return PaymentResponse(False, None, None, None, payment_error)
 
             checking_id = base64.b64decode(data["payment_hash"]).hex()
             fee_msat = int(data["payment_route"]["total_fees_msat"])
             preimage = base64.b64decode(data["payment_preimage"]).hex()
             return PaymentResponse(True, checking_id, fee_msat, preimage, None)
+        except KeyError as exc:
+            logger.warning(exc)
+            return PaymentResponse(
+                None, None, None, None, "Server error: 'missing required fields'"
+            )
         except json.JSONDecodeError:
             return PaymentResponse(
-                False, None, None, None, "Server error: 'invalid json response'"
+                None, None, None, None, "Server error: 'invalid json response'"
+            )
+        except Exception as exc:
+            logger.warning(f"LndRestWallet pay_invoice POST error: {exc}.")
+            return PaymentResponse(
+                None, None, None, None, f"Unable to connect to {self.endpoint}."
             )
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:

lnbits/wallets/phoenixd.py

@@ -144,11 +144,11 @@ class PhoenixdWallet(Wallet):
             data = r.json()
 
             if "routingFeeSat" not in data and "reason" in data:
-                return PaymentResponse(False, None, None, None, data["reason"])
+                return PaymentResponse(None, None, None, None, data["reason"])
 
             if r.is_error or "paymentHash" not in data:
                 error_message = data["message"] if "message" in data else r.text
-                return PaymentResponse(False, None, None, None, error_message)
+                return PaymentResponse(None, None, None, None, error_message)
 
             checking_id = data["paymentHash"]
             fee_msat = -int(data["routingFeeSat"])
@@ -158,17 +158,17 @@ class PhoenixdWallet(Wallet):
 
         except json.JSONDecodeError:
             return PaymentResponse(
-                False, None, None, None, "Server error: 'invalid json response'"
+                None, None, None, None, "Server error: 'invalid json response'"
             )
         except KeyError:
             return PaymentResponse(
-                False, None, None, None, "Server error: 'missing required fields'"
+                None, None, None, None, "Server error: 'missing required fields'"
             )
         except Exception as exc:
             logger.info(f"Failed to pay invoice {bolt11}")
             logger.warning(exc)
             return PaymentResponse(
-                False, None, None, None, f"Unable to connect to {self.endpoint}."
+                None, None, None, None, f"Unable to connect to {self.endpoint}."
             )
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
@@ -189,6 +189,7 @@ class PhoenixdWallet(Wallet):
             return PaymentPendingStatus()
 
     async def get_payment_status(self, checking_id: str) -> PaymentStatus:
+        # TODO: how can we detect a failed payment?
         try:
             r = await self.client.get(f"/payments/outgoing/{checking_id}")
             if r.is_error: