[TEST] workflows, added 10s timeout to lncli subprocess and logging (#1910)

* [FIX] workflows

testing why workflwos unstable instable

* debug run_command

* add timeout to process run

* log lncli output

* try catch json exception in clnrest

* settle does not return a json

* add 10 minutes timeout to regtests if they ever get stuck

* better subprocess handling

* fix race condition

* test 3

* remove

* run nr 4

* increase waits for race conditions

* revert clnrest

* better fiat amount test

.env.example

@@ -69,15 +69,15 @@ LNBITS_SITE_DESCRIPTION="Some description about your service, will display if ti
 LNBITS_THEME_OPTIONS="classic, bitcoin, flamingo, freedom, mint, autumn, monochrome, salvador, cyber"
 # LNBITS_CUSTOM_LOGO="https://lnbits.com/assets/images/logo/logo.svg"
 
-# Choose from LNPayWallet, OpenNodeWallet, ClicheWallet,
-#              LndWallet, LndRestWallet, CoreLightningWallet, EclairWallet,
-#              LnTipsWallet, LNbitsWallet, SparkWallet, FakeWallet,
+# which fundingsources are allowed in the admin ui
+LNBITS_ALLOWED_FUNDING_SOURCES="VoidWallet, FakeWallet, CoreLightningWallet, CoreLightningRestWallet, LndRestWallet, EclairWallet, LndWallet, LnTipsWallet, LNPayWallet, LNbitsWallet, OpenNodeWallet"
+
 LNBITS_BACKEND_WALLET_CLASS=VoidWallet
 # VoidWallet is just a fallback that works without any actual Lightning capabilities,
 # just so you can see the UI before dealing with this file.
 
 # Invoice expiry for LND, CLN, Eclair, LNbits funding sources
-LIGHTNING_INVOICE_EXPIRY=600
+LIGHTNING_INVOICE_EXPIRY=3600
 
 # Set one of these blocks depending on the wallet kind you chose above:
 

.flake8

@@ -1,16 +0,0 @@
-[flake8]
-max-line-length = 150
-exclude = lnbits/wallets/lnd_grpc_files/, lnbits/extensions/
-ignore =
-    # E203 whitespace before ':' black does not like it
-    E203
-    # E402: module level import not at top of file
-    E402,
-    # W503: line break before binary operator
-    W503,
-    # F821: undefined name - should be addressed in future PR
-    F821,
-    # E722 do not use bare 'except' - should be addressed in future PR
-    E722,
-    # flake8-requirements import checks
-    I

.github/actions/prepare/action.yml

@@ -0,0 +1,67 @@
+name: prepare
+
+inputs:
+  python-version:
+    description: "Python Version"
+    required: true
+    default: "3.9"
+  poetry-version:
+    description: "Poetry Version"
+    default: "1.5.1"
+  node-version:
+    description: "Node Version"
+    default: "18.x"
+  npm:
+    description: "use npm"
+    default: false
+    type: boolean
+
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+        # cache poetry install via pip
+        cache: "pip"
+
+    - name: Set up Poetry ${{ inputs.poetry-version }}
+      uses: abatilo/actions-poetry@v2
+      with:
+        poetry-version: ${{ inputs.poetry-version }}
+
+    - name: Setup a local virtual environment (if no poetry.toml file)
+      shell: bash
+      run: |
+        poetry config virtualenvs.create true --local
+        poetry config virtualenvs.in-project true --local
+
+    - uses: actions/cache@v3
+      name: Define a cache for the virtual environment based on the dependencies lock file
+      with:
+        path: ./.venv
+        key: venv-${{ hashFiles('poetry.lock') }}
+
+    - name: Install the project dependencies
+      shell: bash
+      run: poetry install
+
+    - name: Use Node.js ${{ inputs.node-version }}
+      if: ${{ (inputs.npm == 'true') }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ inputs.node-version }}
+
+    - uses: actions/cache@v3
+      if: ${{ (inputs.npm == 'true') }}
+      name: Define a cache for the npm based on the dependencies lock file
+      with:
+        path: ./node_modules
+        key: npm-${{ hashFiles('package-lock.json') }}
+
+    - name: Install npm packages
+      if: ${{ (inputs.npm == 'true') }}
+      shell: bash
+      run: npm install

.github/workflows/ci.yml

@@ -0,0 +1,47 @@
+name: LNbits CI
+on: [push, pull_request]
+
+jobs:
+
+  lint:
+    uses: ./.github/workflows/lint.yml
+
+  tests:
+    needs: [ lint ]
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+        db-url: ["", "postgres://lnbits:lnbits@0.0.0.0:5432/lnbits"]
+    uses: ./.github/workflows/tests.yml
+    with:
+      python-version: ${{ matrix.python-version }}
+
+  migrations:
+    needs: [ lint ]
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+    uses: ./.github/workflows/tests.yml
+    with:
+      make: test-migration
+      db-name: migration
+
+  openapi:
+    needs: [ lint ]
+    uses: ./.github/workflows/make.yml
+    with:
+      make: openapi
+
+  # docker:
+  #   uses: ./.github/workflows/docker.yml
+
+  regtest:
+    needs: [ lint ]
+    uses: ./.github/workflows/regtest.yml
+    strategy:
+      matrix:
+        python-version: ["3.9"]
+        backend-wallet-class: ["LndRestWallet", "LndWallet", "CoreLightningWallet", "CoreLightningRestWallet", "LNbitsWallet", "EclairWallet"]
+    with:
+      python-version: ${{ matrix.python-version }}
+      backend-wallet-class: ${{ matrix.backend-wallet-class }}

.github/workflows/docker.yml

@@ -0,0 +1,20 @@
+name: docker
+
+on:
+  workflow_call:
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+            context: .
+            push: false
+            tags: lnbitsdocker/lnbits-legend:latest
+            cache-from: type=registry,ref=lnbitsdocker/lnbits-legend:latest
+            cache-to: type=inline

.github/workflows/flake8.yml

@@ -1,27 +0,0 @@
-name: flake8
-
-on: [push, pull_request]
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        run: make flake8

.github/workflows/formatting.yml

@@ -1,33 +0,0 @@
-name: formatting
-
-on: [push, pull_request]
-
-jobs:
-  checks:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install packages
-        run: |
-          poetry install
-          npm install prettier
-      - name: Check black
-        run: make checkblack
-      - name: Check isort
-        run: make checkisort
-      - uses: actions/setup-node@v3
-      - name: Check prettier
-        run: make checkprettier

.github/workflows/lint.yml

@@ -0,0 +1,27 @@
+name: lint
+on:
+  workflow_call:
+
+jobs:
+  black:
+    uses: ./.github/workflows/make.yml
+    with:
+      make: checkblack
+  ruff:
+    uses: ./.github/workflows/make.yml
+    with:
+      make: checkruff
+  mypy:
+    uses: ./.github/workflows/make.yml
+    with:
+      make: mypy
+  pyright:
+    uses: ./.github/workflows/make.yml
+    with:
+      make: pyright
+      npm: true
+  prettier:
+    uses: ./.github/workflows/make.yml
+    with:
+      make: prettier
+      npm: true

.github/workflows/make.yml

@@ -0,0 +1,31 @@
+name: make
+
+on:
+  workflow_call:
+    inputs:
+      make:
+        description: "make command that is run"
+        required: true
+        type: string
+      npm:
+        description: "use npm install"
+        default: false
+        type: boolean
+
+jobs:
+  make:
+    name: ${{ inputs.make }} (${{ matrix.python-version }})
+    strategy:
+      matrix:
+        os-version: ["ubuntu-latest"]
+        python-version: ["3.9", "3.10"]
+        node-version: ["18.x"]
+    runs-on: ${{ matrix.os-version }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/prepare
+        with:
+          python-version: ${{ matrix.python-version }}
+          node-version: ${{ matrix.node-version }}
+          npm: ${{ inputs.npm }}
+      - run: make ${{ inputs.make }}

.github/workflows/migrations.yml

@@ -1,43 +0,0 @@
-name: migrations
-
-on: [pull_request]
-
-jobs:
-  sqlite-to-postgres:
-    runs-on: ubuntu-latest
-    services:
-      postgres:
-        image: postgres:latest
-        env:
-          POSTGRES_USER: lnbits
-          POSTGRES_PASSWORD: lnbits
-          POSTGRES_DB: migration
-        ports:
-          # maps tcp port 5432 on service container to the host
-          - 5432:5432
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Install dependencies
-        run: |
-          poetry install
-          sudo apt install unzip
-      - name: Run migrations
-        run: |
-          make test-migration

.github/workflows/mypy.yml

@@ -1,27 +0,0 @@
-name: mypy
-
-on: [push, pull_request]
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        run: make mypy

.github/workflows/pylint.yml

@@ -1,27 +0,0 @@
-name: pylint
-
-on: [push, pull_request]
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        run: make pylint

.github/workflows/pyright.yml

@@ -1,28 +0,0 @@
-name: pyright
-
-on: [push, pull_request]
-
-jobs:
-  check:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install dependencies
-        run: |
-          poetry install
-          npm install
-      - name: Run tests
-        run: make pyright

.github/workflows/regtest.yml

@@ -1,223 +1,81 @@
 name: regtest
 
-on: [push, pull_request]
+on:
+  workflow_call:
+    inputs:
+      make:
+        default: test
+        type: string
+      python-version:
+        default: "3.9"
+        type: string
+      os-version:
+        default: "ubuntu-latest"
+        type: string
+      backend-wallet-class:
+        required: true
+        type: string
 
 jobs:
-  LndRestWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
+  regtest:
+    runs-on: ${{ inputs.os-version }}
+    timeout-minutes: 10
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
+
+      - name: Set up Docker Buildx
+        if: ${{ inputs.backend-wallet-class == 'LNbitsWallet' }}
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push
+        if: ${{ inputs.backend-wallet-class == 'LNbitsWallet' }}
+        uses: docker/build-push-action@v4
         with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
+            context: .
+            push: false
+            tags: lnbitsdocker/lnbits-legend:latest
+            cache-from: type=registry,ref=lnbitsdocker/lnbits-legend:latest
+            cache-to: type=inline
+
       - name: Setup Regtest
         run: |
-          docker build -t lnbitsdocker/lnbits-legend .
           git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
           cd docker
           chmod +x ./tests
           ./tests
           sudo chmod -R a+rwx .
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
+
+      - uses: ./.github/actions/prepare
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: Create fake admin
+        if: ${{ inputs.backend-wallet-class == 'LNbitsWallet' }}
+        run: docker exec lnbits-legend-lnbits-1 poetry run python tools/create_fake_admin.py
+
+      - name: Run Tests
         env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: LndRestWallet
+          LNBITS_DATABASE_URL: ${{ inputs.db-url }}
+          LNBITS_BACKEND_WALLET_CLASS: ${{ inputs.backend-wallet-class }}
           LND_REST_ENDPOINT: https://localhost:8081/
           LND_REST_CERT: ./docker/data/lnd-3/tls.cert
           LND_REST_MACAROON: ./docker/data/lnd-3/data/chain/bitcoin/regtest/admin.macaroon
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-  LndWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Setup Regtest
-        run: |
-          docker build -t lnbitsdocker/lnbits-legend .
-          git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
-          cd docker
-          chmod +x ./tests
-          ./tests
-          sudo chmod -R a+rwx .
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: LndWallet
           LND_GRPC_ENDPOINT: localhost
           LND_GRPC_PORT: 10009
           LND_GRPC_CERT: docker/data/lnd-3/tls.cert
           LND_GRPC_MACAROON: docker/data/lnd-3/data/chain/bitcoin/regtest/admin.macaroon
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-  CoreLightningWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Setup Regtest
-        run: |
-          docker build -t lnbitsdocker/lnbits-legend .
-          git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
-          cd docker
-          chmod +x ./tests
-          ./tests
-          sudo chmod -R a+rwx .
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: CoreLightningWallet
           CORELIGHTNING_RPC: ./docker/data/clightning-1/regtest/lightning-rpc
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-  LNbitsWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Setup Regtest
-        run: |
-          docker build -t lnbitsdocker/lnbits-legend .
-          git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
-          cd docker
-          chmod +x ./tests
-          ./tests
-          sudo chmod -R a+rwx .
-          docker exec lnbits-legend-lnbits-1 /bin/bash -c "poetry run python tools/create_fake_admin.py"
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: LNbitsWallet
+          CORELIGHTNING_REST_URL: https://localhost:3001
+          CORELIGHTNING_REST_MACAROON: ./docker/data/clightning-2-rest/access.macaroon
+          CORELIGHTNING_REST_CERT: ./docker/data/clightning-2-rest/certificate.pem
           LNBITS_ENDPOINT: http://localhost:5001
           LNBITS_KEY: "d08a3313322a4514af75d488bcc27eee"
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          file: ./coverage.xml
-  EclairWallet:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Setup Regtest
-        run: |
-          docker build -t lnbitsdocker/lnbits-legend .
-          git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
-          cd docker
-          chmod +x ./tests
-          ./tests
-          sudo chmod -R a+rwx .
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        env:
-          PYTHONUNBUFFERED: 1
-          PORT: 5123
-          LNBITS_DATA_FOLDER: ./data
-          LNBITS_BACKEND_WALLET_CLASS: EclairWallet
           ECLAIR_URL: http://127.0.0.1:8082
           ECLAIR_PASS: lnbits
-        run: |
-          sudo chmod -R a+rwx . && rm -rf ./data && mkdir -p ./data
-          make test-real-wallet
+        run: make test-real-wallet
+
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           file: ./coverage.xml

.github/workflows/tests.yml

@@ -1,70 +1,58 @@
 name: tests
 
-on: [push, pull_request]
+on:
+  workflow_call:
+    inputs:
+      make:
+        default: test
+        type: string
+      python-version:
+        default: "3.9"
+        type: string
+      os-version:
+        default: "ubuntu-latest"
+        type: string
+      db-url:
+        default: ""
+        type: string
+      db-name:
+        default: "lnbits"
+        type: string
 
 jobs:
-  sqlite:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
-        with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
-        run: make test
-  postgres:
-    runs-on: ubuntu-latest
+  tests:
+    runs-on: ${{ inputs.os-version }}
+
     services:
       postgres:
         image: postgres:latest
         env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
+          POSTGRES_USER: lnbits
+          POSTGRES_PASSWORD: lnbits
+          POSTGRES_DB: ${{ inputs.db-name }}
         ports:
-          # maps tcp port 5432 on service container to the host
           - 5432:5432
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-        poetry-version: ["1.3.1"]
+
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Poetry ${{ matrix.poetry-version }}
-        uses: abatilo/actions-poetry@v2
+
+      - uses: ./.github/actions/prepare
         with:
-          poetry-version: ${{ matrix.poetry-version }}
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-          cache: "poetry"
-      - name: Install dependencies
-        run: |
-          poetry install
-      - name: Run tests
+          python-version: ${{ inputs.python-version }}
+
+      - name: Run Tests
         env:
-          LNBITS_DATABASE_URL: postgres://postgres:postgres@0.0.0.0:5432/postgres
-        run: make test
+          LNBITS_DATABASE_URL: ${{ inputs.db-url }}
+          LNBITS_BACKEND_WALLET_CLASS: FakeWallet
+        run: make ${{ inputs.make }}
+
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           file: ./coverage.xml

.gitignore

@@ -47,5 +47,5 @@ fly.toml
 lnbits-backup.zip
 
 # Ignore extensions (post installable extension PR)
-extensions/
+extensions
 upgrades/

.pre-commit-config.yaml

@@ -14,22 +14,17 @@ repos:
       - id: mixed-line-ending
       - id: check-case-conflict
   - repo: https://github.com/psf/black
-    rev: 22.6.0
+    rev: 23.7.0
     hooks:
       - id: black
-  - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.0.283
     hooks:
-      - id: isort
-        args: ['--profile', 'black']
+      - id: ruff
+        args: [ --fix, --exit-non-zero-on-fix ]
   - repo: https://github.com/pre-commit/mirrors-prettier
     rev: '50c5478ed9e10bf360335449280cf2a67f4edb7a'
     hooks:
       - id: prettier
         types_or: [css, javascript, html, json]
         args: ['lnbits']
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
-    hooks:
-      - id: flake8
-        entry: poetry run flake8

Dockerfile

@@ -8,7 +8,7 @@ RUN curl -sSL https://install.python-poetry.org | python3 -
 ENV PATH="/root/.local/bin:$PATH"
 
 # needed for backups postgresql-client version 14 (pg_dump)
-RUN apt-get install -y wget
+RUN apt-get install -y apt-utils wget
 RUN echo "deb http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 RUN apt-get update
@@ -20,7 +20,6 @@ COPY . .
 
 RUN mkdir data
 
-RUN poetry config virtualenvs.create false
 RUN poetry install --only main
 
 ENV LNBITS_PORT="5000"

Makefile

@@ -2,9 +2,9 @@
 
 all: format check
 
-format: prettier isort black
+format: prettier black ruff
 
-check: mypy pyright pylint flake8 checkisort checkblack checkprettier
+check: mypy pyright checkblack checkruff checkprettier
 
 prettier:
 	poetry run ./node_modules/.bin/prettier --write lnbits
@@ -12,20 +12,17 @@ prettier:
 pyright:
 	poetry run ./node_modules/.bin/pyright
 
-black:
-	poetry run black .
-
-flake8:
-	poetry run flake8
-
 mypy:
 	poetry run mypy
 
-isort:
-	poetry run isort .
+black:
+	poetry run black .
 
-pylint:
-	poetry run pylint *.py lnbits/ tools/ tests/
+ruff:
+	poetry run ruff check . --fix
+
+checkruff:
+	poetry run ruff check .
 
 checkprettier:
 	poetry run ./node_modules/.bin/prettier --check lnbits
@@ -33,8 +30,8 @@ checkprettier:
 checkblack:
 	poetry run black --check .
 
-checkisort:
-	poetry run isort --check-only .
+dev:
+	poetry run lnbits --reload
 
 test:
 	LNBITS_BACKEND_WALLET_CLASS="FakeWallet" \
@@ -69,6 +66,17 @@ test-migration:
 migration:
 	poetry run python tools/conv.py
 
+openapi:
+	LNBITS_BACKEND_WALLET_CLASS="FakeWallet" \
+	LNBITS_DATA_FOLDER="./tests/data" \
+	PYTHONUNBUFFERED=1 \
+	HOST=0.0.0.0 \
+	PORT=5003 \
+	poetry run lnbits &
+	sleep 15
+	curl -s http://0.0.0.0:5003/openapi.json | poetry run openapi-spec-validator --errors=all -
+	# kill -9 %1
+
 bak:
 	# LNBITS_DATABASE_URL=postgres://postgres:postgres@0.0.0.0:5432/postgres
 	#

README.md

@@ -31,7 +31,7 @@ LNbits is inspired by all the great work of [opennode.com](https://www.opennode.
 
 ## Running LNbits
 
-See the [install guide](https://github.com/lnbits/lnbits/wiki/LNbits-Documentation#install-options) for details on installation and setup.
+See the [install guide](https://github.com/lnbits/lnbits/blob/main/docs/guide/installation.md) for details on installation and setup.
 
 ## LNbits as an account system
 

docs/devs/development.md

@@ -12,6 +12,21 @@ For developers
 Thanks for contributing :)
 
 
+Run
+=====
+
+This starts the lnbits uvicorn server
+```bash
+poetry run lnbits
+```
+
+This starts the lnbits uvicorn with hot reloading.
+```bash
+make dev
+# or
+poetry run lnbits --reload
+```
+
 Precommit hooks
 =====
 

docs/guide/faq.md

@@ -20,7 +20,7 @@ allow-self-payment=1
 
 </details>
   
-<details><summary>Funding source only available via tor (e.g. Umbrel)</summary>
+<details><summary>Funding source only available via tor (e.g. Start9 or Umbrel)</summary>
   <p>If you want your setup to stay behind tor then only apps, pos and wallets that have tor activated can communicate with your wallets. Most likely you will have trouble when people try to redeem your voucher through onion or when importing your lnbits wallets into a wallet-app that doesnt support tor. If you plan to let LNbits wallets interact with plain internet shops and services you should consider <a href="https://github.com/TrezorHannes/Dual-LND-Hybrid-VPS">setting up hybrid mode for your node</a>.</p>
 </details>
     
@@ -73,7 +73,7 @@ allow-self-payment=1
         You need to have a domain and to be able to configure a CNAME for your DNS record as well as generate a subdomain dedicated to your LNbits instance like eg. lnbits.yourdomain.com. 
         You also need access to your internet router to open the https port (usually 443) and forward it your LNbits IP within your LAN (usually 80). The ports might depend on your node implementation if those ports do not work please ask for them in a help group of your node supplier.</li>
        <li>You can also follow the Apache installation option, explained in the <a href="https://docs.lnbits.org/guide/installation.html#running-behind-an-apache2-reverse-proxy-over-https">LNbits installation manual</a>.</li>
-       <li>If you run LNbits from a bundle node (Umbrel, myNode, Embassy, Raspiblitz etc), you can follow <a href="https://github.com/TrezorHannes/vps-lnbits">this extensive guide</a> with many options to switch your Tor only LNbits into a clearnet LNbits. For Citadel there is a HTTPS Option in your manual to activate https for LNbits in the newest version.</li>
+       <li>If you run LNbits from a bundle node (myNode, Start9, Umbrel,Raspiblitz etc), you can follow <a href="https://github.com/TrezorHannes/vps-lnbits">this extensive guide</a> with many options to switch your Tor only LNbits into a clearnet LNbits. For Citadel there is a HTTPS Option in your manual to activate https for LNbits in the newest version.</li>
        </ul>
     </details>
    </li>

docs/logos/lnbits-full-inverse.svg

@@ -0,0 +1,11 @@
+<svg width="3032" height="902" viewBox="0 0 3032 902" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M158.566 493.857L1 901L450.49 355.202H264.831L501.791 1H187.881L36.4218 493.857H158.566Z" fill="#FF1FE1" stroke="black"/>
+<path d="M616.041 764.863V134.611H741.647V645.115H1023.41V764.863H616.041Z" fill="#F8F8F8"/>
+<path d="M1079.99 764.863V134.611H1198.81L1478.31 543.014V134.611H1600.52V764.863H1478.31L1198.81 360.241V764.863H1079.99Z" fill="#F8F8F8"/>
+<path d="M616.041 764.863V134.611H741.647V645.115H1023.41V764.863H616.041Z" stroke="black"/>
+<path d="M1079.99 764.863V134.611H1198.81L1478.31 543.014V134.611H1600.52V764.863H1478.31L1198.81 360.241V764.863H1079.99Z" stroke="black"/>
+<path d="M1693.87 768.522V148.458H1764.1V377.349H1770.05C1775.21 369.275 1782.35 358.981 1791.47 346.467C1800.8 333.751 1814.09 322.448 1831.35 312.557C1848.81 302.465 1872.41 297.419 1902.17 297.419C1940.66 297.419 1974.58 307.208 2003.94 326.787C2033.3 346.366 2056.22 374.119 2072.68 410.048C2089.15 445.976 2097.38 488.363 2097.38 537.209C2097.38 586.459 2089.15 629.149 2072.68 665.279C2056.22 701.207 2033.4 729.061 2004.24 748.842C1975.08 768.421 1941.45 778.21 1903.36 778.21C1874 778.21 1850.49 773.265 1832.84 763.375C1815.18 753.283 1801.59 741.878 1792.07 729.162C1782.55 716.244 1775.21 705.547 1770.05 697.069H1761.72V768.522H1693.87ZM1762.91 535.998C1762.91 571.119 1767.96 602.102 1778.08 628.947C1788.2 655.59 1802.98 676.481 1822.42 691.619C1841.86 706.556 1865.67 714.024 1893.84 714.024C1923.2 714.024 1947.7 706.152 1967.34 690.408C1987.18 674.463 2002.06 653.067 2011.98 626.222C2022.1 599.175 2027.15 569.1 2027.15 535.998C2027.15 503.299 2022.19 473.83 2012.28 447.59C2002.55 421.149 1987.78 400.258 1967.94 384.918C1948.3 369.376 1923.6 361.605 1893.84 361.605C1865.27 361.605 1841.27 368.972 1821.83 383.707C1802.38 398.24 1787.7 418.626 1777.78 444.866C1767.87 470.903 1762.91 501.281 1762.91 535.998Z" fill="white"/>
+<path d="M2204.81 768.522V303.474H2275.04V768.522H2204.81ZM2240.52 225.966C2226.83 225.966 2215.03 221.223 2205.11 211.736C2195.39 202.25 2190.52 190.846 2190.52 177.524C2190.52 164.202 2195.39 152.798 2205.11 143.311C2215.03 133.825 2226.83 129.081 2240.52 129.081C2254.21 129.081 2265.91 133.825 2275.63 143.311C2285.55 152.798 2290.51 164.202 2290.51 177.524C2290.51 190.846 2285.55 202.25 2275.63 211.736C2265.91 221.223 2254.21 225.966 2240.52 225.966Z" fill="white"/>
+<path d="M2602.45 303.474V364.027H2365.57V303.474H2602.45ZM2434.61 192.057H2504.84V635.305C2504.84 655.489 2507.72 670.628 2513.47 680.72C2519.42 690.61 2526.96 697.271 2536.09 700.702C2545.41 703.932 2555.23 705.547 2565.55 705.547C2573.28 705.547 2579.63 705.143 2584.59 704.335C2589.55 703.326 2593.52 702.519 2596.49 701.913L2610.78 766.1C2606.02 767.916 2599.37 769.733 2590.84 771.549C2582.31 773.568 2571.5 774.577 2558.41 774.577C2538.57 774.577 2519.12 770.237 2500.08 761.558C2481.23 752.879 2465.56 739.658 2453.06 721.896C2440.76 704.134 2434.61 681.729 2434.61 654.682V192.057Z" fill="white"/>
+<path d="M3023.67 407.625L2960.58 425.791C2956.61 415.094 2950.76 404.699 2943.02 394.607C2935.49 384.313 2925.17 375.835 2912.08 369.174C2898.98 362.513 2882.22 359.183 2861.79 359.183C2833.81 359.183 2810.5 365.743 2791.85 378.863C2773.4 391.781 2764.18 408.231 2764.18 428.214C2764.18 445.976 2770.53 460.004 2783.22 470.298C2795.92 480.592 2815.76 489.17 2842.74 496.033L2910.59 512.988C2951.46 523.08 2981.91 538.521 3001.94 559.311C3021.98 579.899 3032 606.441 3032 638.938C3032 665.581 3024.46 689.399 3009.38 710.391C2994.51 731.383 2973.67 747.934 2946.89 760.044C2920.11 772.155 2888.96 778.21 2853.45 778.21C2806.83 778.21 2768.25 767.916 2737.7 747.328C2707.14 726.74 2687.8 696.665 2679.67 657.104L2746.33 640.149C2752.67 665.178 2764.68 683.949 2782.33 696.464C2800.19 708.978 2823.5 715.235 2852.26 715.235C2885 715.235 2910.99 708.17 2930.23 694.041C2949.67 679.711 2959.39 662.554 2959.39 642.571C2959.39 626.424 2953.84 612.9 2942.73 602.001C2931.62 590.899 2914.56 582.624 2891.54 577.174L2815.36 559.008C2773.5 548.916 2742.75 533.273 2723.11 512.08C2703.67 490.684 2693.95 463.94 2693.95 431.847C2693.95 405.607 2701.19 382.395 2715.67 362.211C2730.36 342.026 2750.29 326.182 2775.49 314.676C2800.88 303.171 2829.65 297.419 2861.79 297.419C2907.02 297.419 2942.53 307.511 2968.32 327.695C2994.31 347.88 3012.76 374.523 3023.67 407.625Z" fill="white"/>
+</svg>

docs/logos/lnbits-full.svg

@@ -1 +1,11 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 101.041 30.0001"><defs><path id="a" d="M33.2619 148.1667h154.2143v68.7917H33.2619z"/></defs><g fill="#1f2234" aria-label="LNbits" font-family="sans-serif" font-size=".3095" font-weight="400" letter-spacing=".0031" style="line-height:1.25;white-space:pre;shape-inside:url(#a)" transform="matrix(72.4607 0 0 72.4607 -2399.2814 -10741.3589)"><g transform="matrix(.00244 0 0 .00244 33.0708 148.1594)"><circle cx="101.2976" cy="116.4167" r="84.6667" fill="#673ab7" fill-rule="evenodd"/><path fill="#eee" d="M79.1105 71.9667v49.0613h13.3803v40.141l31.2208-53.5213h-17.8404l17.8404-35.681z"/></g><g font-family="roboto"><path d="M33.6755 148.518h.0962v.0364h-.1416v-.22h.0454zM33.9796 148.5544h-.0453l-.0883-.1448v.1448h-.0453v-.22h.0453l.0884.145v-.145h.0452z" font-weight="700" style="-inkscape-font-specification:'roboto Bold'"/><path d="M34.1625 148.4744q0 .0375-.0172.0603-.0172.0227-.0462.0227-.031 0-.048-.0219l-.0014.019h-.0256v-.2322h.028v.0866q.0168-.021.0466-.021.0298 0 .0467.0225.017.0225.017.0617zm-.028-.003q0-.0286-.011-.044-.011-.0157-.0317-.0157-.0277 0-.0398.0257v.0707q.0129.0257.04.0257.0202 0 .0314-.0156.0112-.0156.0112-.0469zM34.231 148.5544h-.028v-.1635h.028zm-.0302-.2069q0-.007.0041-.0115.0042-.005.0124-.005.0082 0 .0124.005.0042.005.0042.0115 0 .007-.0042.0113-.0042.005-.0124.005-.0082 0-.0124-.005-.004-.005-.004-.0113zM34.3167 148.3513v.0396h.0305v.0216h-.0305v.1014q0 .01.004.0148.0042.005.014.005.0048 0 .0133-.002v.0227q-.011.003-.0215.003-.0187 0-.0283-.0113-.0095-.0113-.0095-.0322v-.1014h-.0297v-.0216h.0297v-.0396zM34.478 148.511q0-.0113-.0086-.0175-.0085-.006-.0298-.0109-.0211-.005-.0337-.0109-.0124-.006-.0184-.015t-.006-.0209q0-.02.017-.034.017-.0139.0435-.0139.0278 0 .045.0144.0174.0144.0174.0367h-.028q0-.0115-.0099-.0198-.0097-.008-.0245-.008-.0152 0-.0238.007-.0086.007-.0086.0174 0 .0101.008.0153.008.005.0288.01.021.005.034.0112.013.007.0192.0157.0063.009.0063.0222 0 .022-.0175.0352-.0175.0131-.0455.0131-.0197 0-.0348-.007-.015-.007-.0237-.0193-.0085-.0125-.0085-.027h.028q.0007.014.0111.0223.0106.008.0279.008.0158 0 .0253-.006.0097-.006.0097-.0172z" style="-inkscape-font-specification:roboto"/></g></g></svg>
+<svg width="3032" height="902" viewBox="0 0 3032 902" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M158.566 493.857L1 901L450.49 355.202H264.831L501.791 1H187.881L36.4218 493.857H158.566Z" fill="#FF1FE1" stroke="black"/>
+<path d="M616.041 764.863V134.611H741.647V645.115H1023.41V764.863H616.041Z" fill="#1E1E1E"/>
+<path d="M1079.99 764.863V134.611H1198.81L1478.31 543.014V134.611H1600.52V764.863H1478.31L1198.81 360.241V764.863H1079.99Z" fill="#1E1E1E"/>
+<path d="M616.041 764.863V134.611H741.647V645.115H1023.41V764.863H616.041Z" stroke="black"/>
+<path d="M1079.99 764.863V134.611H1198.81L1478.31 543.014V134.611H1600.52V764.863H1478.31L1198.81 360.241V764.863H1079.99Z" stroke="black"/>
+<path d="M1693.87 768.522V148.458H1764.1V377.349H1770.05C1775.21 369.275 1782.35 358.981 1791.47 346.467C1800.8 333.751 1814.09 322.448 1831.35 312.557C1848.81 302.465 1872.41 297.419 1902.17 297.419C1940.66 297.419 1974.58 307.208 2003.94 326.787C2033.3 346.366 2056.22 374.119 2072.68 410.048C2089.15 445.976 2097.38 488.363 2097.38 537.209C2097.38 586.459 2089.15 629.149 2072.68 665.279C2056.22 701.207 2033.4 729.061 2004.24 748.842C1975.08 768.421 1941.45 778.21 1903.36 778.21C1874 778.21 1850.49 773.265 1832.84 763.375C1815.18 753.282 1801.59 741.878 1792.07 729.162C1782.55 716.244 1775.21 705.547 1770.05 697.069H1761.72V768.522H1693.87ZM1762.91 535.998C1762.91 571.119 1767.96 602.102 1778.08 628.947C1788.2 655.59 1802.98 676.481 1822.42 691.619C1841.86 706.556 1865.67 714.024 1893.84 714.024C1923.2 714.024 1947.7 706.152 1967.34 690.408C1987.18 674.463 2002.06 653.067 2011.98 626.222C2022.1 599.175 2027.15 569.1 2027.15 535.998C2027.15 503.299 2022.2 473.83 2012.28 447.59C2002.55 421.149 1987.78 400.258 1967.94 384.918C1948.3 369.376 1923.6 361.605 1893.84 361.605C1865.27 361.605 1841.27 368.972 1821.83 383.707C1802.38 398.24 1787.7 418.626 1777.78 444.866C1767.87 470.903 1762.91 501.281 1762.91 535.998Z" fill="black"/>
+<path d="M2204.81 768.522V303.474H2275.04V768.522H2204.81ZM2240.52 225.966C2226.83 225.966 2215.03 221.223 2205.11 211.736C2195.39 202.25 2190.52 190.846 2190.52 177.524C2190.52 164.202 2195.39 152.798 2205.11 143.311C2215.03 133.825 2226.83 129.081 2240.52 129.081C2254.21 129.081 2265.91 133.825 2275.63 143.311C2285.55 152.798 2290.51 164.202 2290.51 177.524C2290.51 190.846 2285.55 202.25 2275.63 211.736C2265.91 221.223 2254.21 225.966 2240.52 225.966Z" fill="black"/>
+<path d="M2602.45 303.474V364.027H2365.57V303.474H2602.45ZM2434.61 192.057H2504.84V635.305C2504.84 655.489 2507.72 670.628 2513.47 680.72C2519.42 690.61 2526.96 697.271 2536.09 700.702C2545.41 703.932 2555.23 705.546 2565.55 705.546C2573.28 705.546 2579.63 705.143 2584.59 704.335C2589.55 703.326 2593.52 702.519 2596.5 701.913L2610.78 766.1C2606.02 767.916 2599.37 769.733 2590.84 771.549C2582.31 773.568 2571.5 774.577 2558.41 774.577C2538.57 774.577 2519.12 770.237 2500.08 761.558C2481.23 752.879 2465.56 739.658 2453.06 721.896C2440.76 704.134 2434.61 681.729 2434.61 654.682V192.057Z" fill="black"/>
+<path d="M3023.67 407.625L2960.58 425.791C2956.61 415.094 2950.76 404.699 2943.02 394.607C2935.49 384.313 2925.17 375.835 2912.08 369.174C2898.98 362.513 2882.22 359.183 2861.79 359.183C2833.81 359.183 2810.5 365.743 2791.85 378.863C2773.41 391.781 2764.18 408.231 2764.18 428.213C2764.18 445.976 2770.53 460.004 2783.23 470.298C2795.92 480.592 2815.76 489.17 2842.74 496.033L2910.59 512.988C2951.46 523.08 2981.91 538.521 3001.94 559.311C3021.98 579.899 3032 606.441 3032 638.938C3032 665.581 3024.46 689.399 3009.38 710.391C2994.51 731.383 2973.67 747.934 2946.89 760.044C2920.11 772.155 2888.96 778.21 2853.45 778.21C2806.83 778.21 2768.25 767.916 2737.7 747.328C2707.14 726.74 2687.8 696.665 2679.67 657.104L2746.33 640.149C2752.67 665.178 2764.68 683.949 2782.33 696.464C2800.19 708.978 2823.5 715.235 2852.26 715.235C2885 715.235 2910.99 708.17 2930.23 694.041C2949.67 679.711 2959.39 662.554 2959.39 642.571C2959.39 626.424 2953.84 612.9 2942.73 602.001C2931.62 590.899 2914.56 582.624 2891.54 577.174L2815.36 559.008C2773.5 548.916 2742.75 533.273 2723.11 512.079C2703.67 490.684 2693.95 463.94 2693.95 431.847C2693.95 405.607 2701.19 382.395 2715.67 362.211C2730.36 342.026 2750.29 326.182 2775.49 314.676C2800.88 303.171 2829.65 297.419 2861.79 297.419C2907.02 297.419 2942.53 307.511 2968.32 327.695C2994.31 347.88 3012.76 374.523 3023.67 407.625Z" fill="black"/>
+</svg>

docs/logos/lnbits.svg

@@ -1 +1,19 @@
-<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 30 30"><g transform="translate(-2.9464 -5.625) scale(.17717)"><circle cx="101.2976" cy="116.4167" r="84.6667" fill="#673ab7" fill-rule="evenodd"/><path fill="#eee" d="M79.1105 71.9667v49.0613h13.3803v40.141l31.2208-53.5213h-17.8404l17.8404-35.681z"/></g></svg>
+<svg width="900" height="900" viewBox="0 0 900 900" fill="none" xmlns="http://www.w3.org/2000/svg">
+<g filter="url(#filter0_i_10_29)">
+<circle cx="450" cy="450" r="450" fill="#1E1E1E"/>
+</g>
+<path d="M387.802 476.423L281.613 730.887L584.54 389.763H459.418L619.113 168.387H407.558L305.485 476.423H387.802Z" fill="#FF1FE1"/>
+<defs>
+<filter id="filter0_i_10_29" x="-0.1" y="-0.1" width="900.1" height="900.1" filterUnits="userSpaceOnUse" color-interpolation-filters="sRGB">
+<feFlood flood-opacity="0" result="BackgroundImageFix"/>
+<feBlend mode="normal" in="SourceGraphic" in2="BackgroundImageFix" result="shape"/>
+<feColorMatrix in="SourceAlpha" type="matrix" values="0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 127 0" result="hardAlpha"/>
+<feMorphology radius="0.6" operator="dilate" in="SourceAlpha" result="effect1_innerShadow_10_29"/>
+<feOffset dx="-1" dy="-0.4"/>
+<feGaussianBlur stdDeviation="0.35"/>
+<feComposite in2="hardAlpha" operator="arithmetic" k2="-1" k3="1"/>
+<feColorMatrix type="matrix" values="0 0 0 0 1 0 0 0 0 0.121569 0 0 0 0 0.882353 0 0 0 1 0"/>
+<feBlend mode="normal" in2="shape" result="effect1_innerShadow_10_29"/>
+</filter>
+</defs>
+</svg>

lnbits/app.py

@@ -14,7 +14,6 @@ from typing import Callable, List
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.middleware.gzip import GZipMiddleware
 from fastapi.staticfiles import StaticFiles
 from loguru import logger
 from slowapi import Limiter
@@ -27,9 +26,10 @@ from lnbits.core.services import websocketUpdater
 from lnbits.core.tasks import (  # register_watchdog,; unregister_watchdog,
     register_killswitch,
     register_task_listeners,
-    unregister_killswitch,
 )
 from lnbits.settings import settings
+from lnbits.tasks import cancel_all_tasks, create_permanent_task
+from lnbits.utils.cache import cache
 from lnbits.wallets import get_wallet_class, set_wallet_class
 
 from .commands import db_versions, load_disabled_extension_list, migrate_databases
@@ -44,6 +44,7 @@ from .core.views.generic import core_html_routes
 from .extension_manager import Extension, InstallableExtension, get_valid_extensions
 from .helpers import template_renderer
 from .middleware import (
+    CustomGZipMiddleware,
     ExtensionsRedirectMiddleware,
     InstalledExtensionMiddleware,
     add_ip_block_middleware,
@@ -51,7 +52,6 @@ from .middleware import (
 )
 from .requestvars import g
 from .tasks import (
-    catch_everything_and_restart,
     check_pending_payments,
     internal_invoice_listener,
     invoice_listener,
@@ -62,8 +62,11 @@ from .tasks import (
 def create_app() -> FastAPI:
     configure_logger()
     app = FastAPI(
-        title="LNbits API",
-        description="API for LNbits, the free and open source bitcoin wallet and accounts system with plugins.",
+        title=settings.lnbits_title,
+        description=(
+            "API for LNbits, the free and open source bitcoin wallet and "
+            "accounts system with plugins."
+        ),
         version=settings.version,
         license_info={
             "name": "MIT License",
@@ -84,7 +87,9 @@ def create_app() -> FastAPI:
         CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]
     )
 
-    app.add_middleware(GZipMiddleware, minimum_size=1000)
+    app.add_middleware(
+        CustomGZipMiddleware, minimum_size=1000, exclude_paths=["/api/v1/payments/sse"]
+    )
 
     # order of these two middlewares is important
     app.add_middleware(InstalledExtensionMiddleware)
@@ -107,7 +112,9 @@ async def check_funding_source() -> None:
     original_sigint_handler = signal.getsignal(signal.SIGINT)
 
     def signal_handler(signal, frame):
-        logger.debug("SIGINT received, terminating LNbits.")
+        logger.debug(
+            f"SIGINT received, terminating LNbits. signal: {signal}, frame: {frame}"
+        )
         sys.exit(1)
 
     signal.signal(signal.SIGINT, signal_handler)
@@ -129,17 +136,16 @@ async def check_funding_source() -> None:
                 break
 
             logger.error(
-                f"The backend for {WALLET.__class__.__name__} isn't working properly: '{error_message}'",
+                f"The backend for {WALLET.__class__.__name__} isn't "
+                f"working properly: '{error_message}'",
                 RuntimeWarning,
             )
-        except:
+        except Exception as e:
+            logger.error(f"Error connecting to {WALLET.__class__.__name__}: {e}")
             pass
 
         if settings.lnbits_admin_ui and retry_counter == timeout:
-            logger.warning(
-                f"Fallback to VoidWallet, because the backend for {WALLET.__class__.__name__} isn't working properly"
-            )
-            set_wallet_class("VoidWallet")
+            set_void_wallet_class()
             WALLET = get_wallet_class()
             break
         else:
@@ -150,16 +156,25 @@ async def check_funding_source() -> None:
     signal.signal(signal.SIGINT, original_sigint_handler)
 
     logger.info(
-        f"✔️ Backend {WALLET.__class__.__name__} connected and with a balance of {balance} msat."
+        f"✔️ Backend {WALLET.__class__.__name__} connected "
+        f"and with a balance of {balance} msat."
     )
 
 
+def set_void_wallet_class():
+    logger.warning(
+        "Fallback to VoidWallet, because the backend for "
+        f"{settings.lnbits_backend_wallet_class} isn't working properly"
+    )
+    set_wallet_class("VoidWallet")
+
+
 async def check_installed_extensions(app: FastAPI):
     """
-    Check extensions that have been installed, but for some reason no longer present in the 'lnbits/extensions' directory.
-    One reason might be a docker-container that was re-created.
-    The 'data' directory (where the '.zip' files live) is expected to persist state.
-    Zips that are missing will be re-downloaded.
+    Check extensions that have been installed, but for some reason no longer present in
+    the 'lnbits/extensions' directory. One reason might be a docker-container that was
+    re-created. The 'data' directory (where the '.zip' files live) is expected to
+    persist state. Zips that are missing will be re-downloaded.
     """
     shutil.rmtree(os.path.join("lnbits", "upgrades"), True)
     await load_disabled_extension_list()
@@ -171,7 +186,8 @@ async def check_installed_extensions(app: FastAPI):
             if not installed:
                 await restore_installed_extension(app, ext)
                 logger.info(
-                    f"✔️ Successfully re-installed extension: {ext.id} ({ext.installed_version})"
+                    "✔️ Successfully re-installed extension: "
+                    f"{ext.id} ({ext.installed_version})"
                 )
         except Exception as e:
             logger.warning(e)
@@ -179,6 +195,10 @@ async def check_installed_extensions(app: FastAPI):
                 f"Failed to re-install extension: {ext.id} ({ext.installed_version})"
             )
 
+    logger.info(f"Installed Extensions ({len(installed_extensions)}):")
+    for ext in installed_extensions:
+        logger.info(f"{ext.id} ({ext.installed_version})")
+
 
 async def build_all_installed_extensions_list() -> List[InstallableExtension]:
     """
@@ -252,7 +272,8 @@ def register_routes(app: FastAPI) -> None:
 
 def register_new_ext_routes(app: FastAPI) -> Callable:
     # Returns a function that registers new routes for an extension.
-    # The returned function encapsulates (creates a closure around) the `app` object but does expose it.
+    # The returned function encapsulates (creates a closure around)
+    # the `app` object but does expose it.
     def register_new_ext_routes_fn(ext: Extension):
         register_ext_routes(app, ext)
 
@@ -319,7 +340,14 @@ def register_startup(app: FastAPI):
             add_ip_block_middleware(app)
 
             # initialize WALLET
-            set_wallet_class()
+            try:
+                set_wallet_class()
+            except Exception as e:
+                logger.error(
+                    f"Error initializing {settings.lnbits_backend_wallet_class}: "
+                    f"{str(e)}"
+                )
+                set_void_wallet_class()
 
             # initialize funding source
             await check_funding_source()
@@ -338,6 +366,9 @@ def register_startup(app: FastAPI):
 def register_shutdown(app: FastAPI):
     @app.on_event("shutdown")
     async def on_shutdown():
+        cancel_all_tasks()
+        # wait a bit to allow them to finish, so that cleanup can run without problems
+        await asyncio.sleep(0.1)
         WALLET = get_wallet_class()
         await WALLET.cleanup()
 
@@ -352,7 +383,7 @@ def initialize_server_logger():
             msg = await serverlog_queue.get()
             await websocketUpdater(super_user_hash, msg)
 
-    asyncio.create_task(update_websocket_serverlog())
+    create_permanent_task(update_websocket_serverlog)
 
     logger.add(
         lambda msg: serverlog_queue.put_nowait(msg),
@@ -380,9 +411,11 @@ def get_db_vendor_name():
     return (
         "PostgreSQL"
         if db_url and db_url.startswith("postgres://")
-        else "CockroachDB"
-        if db_url and db_url.startswith("cockroachdb://")
-        else "SQLite"
+        else (
+            "CockroachDB"
+            if db_url and db_url.startswith("cockroachdb://")
+            else "SQLite"
+        )
     )
 
 
@@ -393,21 +426,14 @@ def register_async_tasks(app):
 
     @app.on_event("startup")
     async def listeners():
-        loop = asyncio.get_event_loop()
-        loop.create_task(catch_everything_and_restart(check_pending_payments))
-        loop.create_task(catch_everything_and_restart(invoice_listener))
-        loop.create_task(catch_everything_and_restart(internal_invoice_listener))
-        await register_task_listeners()
-        # await register_watchdog()
-        await register_killswitch()
+        create_permanent_task(check_pending_payments)
+        create_permanent_task(invoice_listener)
+        create_permanent_task(internal_invoice_listener)
+        create_permanent_task(cache.invalidate_forever)
+        register_task_listeners()
+        register_killswitch()
         # await run_deferred_async() # calle: doesn't do anyting?
 
-    @app.on_event("shutdown")
-    async def stop_listeners():
-        # await unregister_watchdog()
-        await unregister_killswitch()
-        pass
-
 
 def register_exception_handlers(app: FastAPI):
     @app.exception_handler(Exception)
@@ -483,25 +509,32 @@ def configure_logger() -> None:
     logger.remove()
     log_level: str = "DEBUG" if settings.debug else "INFO"
     formatter = Formatter()
-    logger.add(sys.stderr, level=log_level, format=formatter.format)
+    logger.add(sys.stdout, level=log_level, format=formatter.format)
     logging.getLogger("uvicorn").handlers = [InterceptHandler()]
     logging.getLogger("uvicorn.access").handlers = [InterceptHandler()]
+    logging.getLogger("uvicorn.error").handlers = [InterceptHandler()]
+    logging.getLogger("uvicorn.error").propagate = False
 
 
 class Formatter:
     def __init__(self):
         self.padding = 0
-        self.minimal_fmt: str = "<green>{time:YYYY-MM-DD HH:mm:ss.SS}</green> | <level>{level}</level> | <level>{message}</level>\n"
+        self.minimal_fmt = (
+            "<green>{time:YYYY-MM-DD HH:mm:ss.SS}</green> | <level>{level}</level> | "
+            "<level>{message}</level>\n"
+        )
         if settings.debug:
-            self.fmt: str = (
-                "<green>{time:YYYY-MM-DD HH:mm:ss.SS}</green> | <level>{level: <4}</level> | "
-                "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> | <level>{message}</level>\n"
+            self.fmt = (
+                "<green>{time:YYYY-MM-DD HH:mm:ss.SS}</green> | "
+                "<level>{level: <4}</level> | "
+                "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> | "
+                "<level>{message}</level>\n"
             )
         else:
-            self.fmt: str = self.minimal_fmt
+            self.fmt = self.minimal_fmt
 
     def format(self, record):
-        function = "{function}".format(**record)  # pylint: disable=C0209
+        function = "{function}".format(**record)
         if function == "emit":  # uvicorn logs
             return self.minimal_fmt
         return self.fmt

lnbits/bolt11.py

@@ -182,7 +182,6 @@ def lnencode(addr, privkey):
     tags_set = set()
 
     for k, v in addr.tags:
-
         # BOLT #11:
         #
         # A writer MUST NOT include more than one `d`, `h`, `n` or `x` fields,

lnbits/commands.py

@@ -13,8 +13,40 @@ from .db import COCKROACH, POSTGRES, SQLITE
 from .extension_manager import get_valid_extensions
 
 
-@click.command("migrate")
-def db_migrate():
+@click.group()
+def command_group():
+    """
+    Python CLI for LNbits
+    """
+
+
+@click.command("superuser")
+def superuser():
+    """Prints the superuser"""
+    with open(".super_user", "r") as file:
+        print(f"http://{settings.host}:{settings.port}/wallet?usr={file.readline()}")
+
+
+@click.command("delete-settings")
+def delete_settings():
+    """Deletes the settings"""
+
+    async def wrap():
+        async with core_db.connect() as conn:
+            await conn.execute("DELETE from settings")
+
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(wrap())
+
+
+@click.command("database-migrate")
+def database_migrate():
+    """Migrate databases"""
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(migrate_databases())
+
+
+async def db_migrate():
     asyncio.create_task(migrate_databases())
 
 
@@ -29,7 +61,8 @@ async def migrate_databases():
             )
         elif conn.type in {POSTGRES, COCKROACH}:
             exists = await conn.fetchone(
-                "SELECT * FROM information_schema.tables WHERE table_schema = 'public' AND table_name = 'dbversions'"
+                "SELECT * FROM information_schema.tables WHERE table_schema = 'public'"
+                " AND table_name = 'dbversions'"
             )
 
         if not exists:
@@ -46,13 +79,33 @@ async def migrate_databases():
     logger.info("✔️ All migrations done.")
 
 
+@click.command("database-versions")
+def database_versions():
+    """Show current database versions"""
+    loop = asyncio.get_event_loop()
+    loop.run_until_complete(db_versions())
+
+
 async def db_versions():
+    """Show current database versions"""
     async with core_db.connect() as conn:
-        current_versions = await get_dbversions(conn)
-        return current_versions
+        return await get_dbversions(conn)
 
 
 async def load_disabled_extension_list() -> None:
     """Update list of extensions that have been explicitly disabled"""
     inactive_extensions = await get_inactive_extensions()
     settings.lnbits_deactivated_extensions += inactive_extensions
+
+
+def main():
+    """main function"""
+    command_group.add_command(superuser)
+    command_group.add_command(delete_settings)
+    command_group.add_command(database_migrate)
+    command_group.add_command(database_versions)
+    command_group()
+
+
+if __name__ == "__main__":
+    main()

lnbits/core/__init__.py

@@ -5,11 +5,11 @@ from lnbits.db import Database
 
 db = Database("database")
 
-core_app: APIRouter = APIRouter()
+core_app: APIRouter = APIRouter(tags=["Core"])
 
 core_app_extra: CoreAppExtra = CoreAppExtra()
 
-from .views.admin_api import *  # noqa: F401,F403
-from .views.api import *  # noqa: F401,F403
-from .views.generic import *  # noqa: F401,F403
-from .views.public_api import *  # noqa: F401,F403
+from .views.admin_api import *
+from .views.api import *
+from .views.generic import *
+from .views.public_api import *

lnbits/core/crud.py

@@ -7,6 +7,7 @@ from uuid import UUID, uuid4
 import shortuuid
 
 from lnbits import bolt11
+from lnbits.core.models import WalletType
 from lnbits.db import Connection, Database, Filters, Page
 from lnbits.extension_manager import InstallableExtension
 from lnbits.settings import AdminSettings, EditableSettings, SuperSettings, settings
@@ -57,7 +58,9 @@ async def get_user(user_id: str, conn: Optional[Connection] = None) -> Optional[
         )
         wallets = await (conn or db).fetchall(
             """
-            SELECT *, COALESCE((SELECT balance FROM balances WHERE wallet = wallets.id), 0) AS balance_msat
+            SELECT *, COALESCE((
+                SELECT balance FROM balances WHERE wallet = wallets.id
+            ), 0) AS balance_msat
             FROM wallets
             WHERE "user" = ?
             """,
@@ -88,9 +91,9 @@ async def add_installed_extension(
     conn: Optional[Connection] = None,
 ) -> None:
     meta = {
-        "installed_release": dict(ext.installed_release)
-        if ext.installed_release
-        else None,
+        "installed_release": (
+            dict(ext.installed_release) if ext.installed_release else None
+        ),
         "dependencies": ext.dependencies,
     }
 
@@ -98,9 +101,11 @@ async def add_installed_extension(
 
     await (conn or db).execute(
         """
-        INSERT INTO installed_extensions (id, version, name, short_description, icon, stars, meta) VALUES (?, ?, ?, ?, ?, ?, ?)
-        ON CONFLICT (id) DO
-        UPDATE SET (version, name, active, short_description, icon, stars, meta) = (?, ?, ?, ?, ?, ?, ?)
+        INSERT INTO installed_extensions
+        (id, version, name, short_description, icon, stars, meta)
+        VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (id) DO UPDATE SET
+        (version, name, active, short_description, icon, stars, meta) =
+        (?, ?, ?, ?, ?, ?, ?)
         """,
         (
             ext.id,
@@ -233,15 +238,25 @@ async def create_wallet(
 
 
 async def update_wallet(
-    wallet_id: str, new_name: str, conn: Optional[Connection] = None
+    wallet_id: str,
+    name: Optional[str] = None,
+    currency: Optional[str] = None,
+    conn: Optional[Connection] = None,
 ) -> Optional[Wallet]:
+    set_clause = []
+    values = []
+    if name:
+        set_clause.append("name = ?")
+        values.append(name)
+    if currency is not None:
+        set_clause.append("currency = ?")
+        values.append(currency)
+    values.append(wallet_id)
     await (conn or db).execute(
-        """
-        UPDATE wallets SET
-            name = ?
-        WHERE id = ?
+        f"""
+        UPDATE wallets SET {', '.join(set_clause)} WHERE id = ?
         """,
-        (new_name, wallet_id),
+        tuple(values),
     )
     wallet = await get_wallet(wallet_id=wallet_id, conn=conn)
     assert wallet, "updated created wallet couldn't be retrieved"
@@ -269,9 +284,8 @@ async def get_wallet(
 ) -> Optional[Wallet]:
     row = await (conn or db).fetchone(
         """
-        SELECT *, COALESCE((SELECT balance FROM balances WHERE wallet = wallets.id), 0) AS balance_msat
-        FROM wallets
-        WHERE id = ?
+        SELECT *, COALESCE((SELECT balance FROM balances WHERE wallet = wallets.id), 0)
+        AS balance_msat FROM wallets WHERE id = ?
         """,
         (wallet_id,),
     )
@@ -280,13 +294,14 @@ async def get_wallet(
 
 
 async def get_wallet_for_key(
-    key: str, key_type: str = "invoice", conn: Optional[Connection] = None
+    key: str,
+    key_type: WalletType = WalletType.invoice,
+    conn: Optional[Connection] = None,
 ) -> Optional[Wallet]:
     row = await (conn or db).fetchone(
         """
-        SELECT *, COALESCE((SELECT balance FROM balances WHERE wallet = wallets.id), 0) AS balance_msat
-        FROM wallets
-        WHERE adminkey = ? OR inkey = ?
+        SELECT *, COALESCE((SELECT balance FROM balances WHERE wallet = wallets.id), 0)
+        AS balance_msat FROM wallets WHERE adminkey = ? OR inkey = ?
         """,
         (key, key),
     )
@@ -294,7 +309,7 @@ async def get_wallet_for_key(
     if not row:
         return None
 
-    if key_type == "admin" and row["adminkey"] != key:
+    if key_type == WalletType.admin and row["adminkey"] != key:
         return None
 
     return Wallet(**row)
@@ -514,13 +529,13 @@ async def create_payment(
     conn: Optional[Connection] = None,
 ) -> Payment:
     # todo: add this when tests are fixed
-    # previous_payment = await get_wallet_payment(wallet_id, payment_hash, conn=conn)
-    # assert previous_payment is None, "Payment already exists"
+    previous_payment = await get_wallet_payment(wallet_id, payment_hash, conn=conn)
+    assert previous_payment is None, "Payment already exists"
 
     try:
         invoice = bolt11.decode(payment_request)
         expiration_date = datetime.datetime.fromtimestamp(invoice.date + invoice.expiry)
-    except:
+    except Exception:
         # assume maximum bolt11 expiry of 31 days to be on the safe side
         expiration_date = datetime.datetime.now() + datetime.timedelta(days=31)
 
@@ -541,9 +556,11 @@ async def create_payment(
             pending,
             memo,
             fee,
-            json.dumps(extra)
-            if extra and extra != {} and type(extra) is dict
-            else None,
+            (
+                json.dumps(extra)
+                if extra and extra != {} and type(extra) is dict
+                else None
+            ),
             webhook,
             db.datetime_to_timestamp(expiration_date),
         ),
@@ -605,7 +622,8 @@ async def update_payment_extra(
 ) -> None:
     """
     Only update the `extra` field for the payment.
-    Old values in the `extra` JSON object will be kept unless the new `extra` overwrites them.
+    Old values in the `extra` JSON object will be kept
+    unless the new `extra` overwrites them.
     """
 
     amount_clause = "AND amount < 0" if outgoing else "AND amount > 0"
@@ -659,7 +677,10 @@ async def check_internal(
 async def check_internal_pending(
     payment_hash: str, conn: Optional[Connection] = None
 ) -> bool:
-    """Returns False if the internal payment is not pending anymore (and thus paid), otherwise True"""
+    """
+    Returns False if the internal payment is not pending anymore
+    (and thus paid), otherwise True
+    """
     row = await (conn or db).fetchone(
         """
         SELECT pending FROM apipayments

lnbits/core/helpers.py

@@ -51,7 +51,8 @@ async def stop_extension_background_work(ext_id: str, user: str):
     """
     Stop background work for extension (like asyncio.Tasks, WebSockets, etc).
     Extensions SHOULD expose a DELETE enpoint at the root level of their API.
-    This function tries first to call the endpoint using `http` and if if fails it tries using `https`.
+    This function tries first to call the endpoint using `http`
+    and if it fails it tries using `https`.
     """
     async with httpx.AsyncClient() as client:
         try:
@@ -71,7 +72,7 @@ def to_valid_user_id(user_id: str) -> UUID:
         raise ValueError("User ID must have at least 128 bits")
     try:
         int(user_id, 16)
-    except:
+    except Exception:
         raise ValueError("Invalid hex string for User ID.")
 
     return UUID(hex=user_id[:32], version=4)

lnbits/core/migrations.py

@@ -239,7 +239,8 @@ async def m007_set_invoice_expiries(db):
                     invoice.date + invoice.expiry
                 )
                 logger.info(
-                    f"Migration: {i+1}/{len(rows)} setting expiry of invoice {invoice.payment_hash} to {expiration_date}"
+                    f"Migration: {i+1}/{len(rows)} setting expiry of invoice"
+                    f" {invoice.payment_hash} to {expiration_date}"
                 )
                 await db.execute(
                     """
@@ -251,7 +252,7 @@ async def m007_set_invoice_expiries(db):
                         checking_id,
                     ),
                 )
-            except:
+            except Exception:
                 continue
     except OperationalError:
         # this is necessary now because it may be the case that this migration will
@@ -301,3 +302,28 @@ async def m010_create_installed_extensions_table(db):
         );
     """
     )
+
+
+async def m011_optimize_balances_view(db):
+    """
+    Make the calculation of the balance a single aggregation
+    over the payments table instead of 2.
+    """
+    await db.execute("DROP VIEW balances")
+    await db.execute(
+        """
+        CREATE VIEW balances AS
+        SELECT wallet, SUM(amount - abs(fee)) AS balance
+        FROM apipayments
+        WHERE (pending = false AND amount > 0) OR amount < 0
+        GROUP BY wallet
+    """
+    )
+
+
+async def m012_add_currency_to_wallet(db):
+    await db.execute(
+        """
+        ALTER TABLE wallets ADD COLUMN currency TEXT
+        """
+    )

lnbits/core/models.py

@@ -3,10 +3,13 @@ import hashlib
 import hmac
 import json
 import time
+from dataclasses import dataclass
+from enum import Enum
 from sqlite3 import Row
 from typing import Callable, Dict, List, Optional
 
 from ecdsa import SECP256k1, SigningKey
+from fastapi import Query
 from lnurl import encode as lnurl_encode
 from loguru import logger
 from pydantic import BaseModel
@@ -24,6 +27,7 @@ class Wallet(BaseModel):
     user: str
     adminkey: str
     inkey: str
+    currency: Optional[str]
     balance_msat: int
 
     @property
@@ -41,7 +45,7 @@ class Wallet(BaseModel):
         url = url_for("/withdraw", external=True, usr=self.user, wal=self.id)
         try:
             return lnurl_encode(url)
-        except:
+        except Exception:
             return ""
 
     def lnurlauth_key(self, domain: str) -> SigningKey:
@@ -58,6 +62,22 @@ class Wallet(BaseModel):
         return await get_standalone_payment(payment_hash)
 
 
+class WalletType(Enum):
+    admin = 0
+    invoice = 1
+    invalid = 2
+
+    # backwards compatibility
+    def __eq__(self, other):
+        return self.value == other
+
+
+@dataclass
+class WalletTypeInfo:
+    wallet_type: WalletType
+    wallet: Wallet
+
+
 class User(BaseModel):
     id: str
     email: Optional[str] = None
@@ -169,6 +189,8 @@ class Payment(FromRowModel):
     async def set_pending(self, pending: bool) -> None:
         from .crud import update_payment_status
 
+        self.pending = pending
+
         await update_payment_status(self.checking_id, pending)
 
     async def check_status(
@@ -179,7 +201,8 @@ class Payment(FromRowModel):
             return PaymentStatus(None)
 
         logger.debug(
-            f"Checking {'outgoing' if self.is_out else 'incoming'} pending payment {self.checking_id}"
+            f"Checking {'outgoing' if self.is_out else 'incoming'} "
+            f"pending payment {self.checking_id}"
         )
 
         WALLET = get_wallet_class()
@@ -193,7 +216,8 @@ class Payment(FromRowModel):
         if self.is_in and status.pending and self.is_expired and self.expiry:
             expiration_date = datetime.datetime.fromtimestamp(self.expiry)
             logger.debug(
-                f"Deleting expired incoming pending payment {self.checking_id}: expired {expiration_date}"
+                f"Deleting expired incoming pending payment {self.checking_id}: "
+                f"expired {expiration_date}"
             )
             await self.delete(conn)
         elif self.is_out and status.failed:
@@ -203,7 +227,8 @@ class Payment(FromRowModel):
             await self.delete(conn)
         elif not status.pending:
             logger.info(
-                f"Marking '{'in' if self.is_in else 'out'}' {self.checking_id} as not pending anymore: {status}"
+                f"Marking '{'in' if self.is_in else 'out'}' "
+                f"{self.checking_id} as not pending anymore: {status}"
             )
             await self.update_status(status, conn=conn)
         return status
@@ -257,3 +282,50 @@ class TinyURL(BaseModel):
     @classmethod
     def from_row(cls, row: Row):
         return cls(**dict(row))
+
+
+class ConversionData(BaseModel):
+    from_: str = "sat"
+    amount: float
+    to: str = "usd"
+
+
+class Callback(BaseModel):
+    callback: str
+
+
+class DecodePayment(BaseModel):
+    data: str
+
+
+class CreateLnurl(BaseModel):
+    description_hash: str
+    callback: str
+    amount: int
+    comment: Optional[str] = None
+    description: Optional[str] = None
+
+
+class CreateInvoice(BaseModel):
+    unit: str = "sat"
+    internal: bool = False
+    out: bool = True
+    amount: float = Query(None, ge=0)
+    memo: Optional[str] = None
+    description_hash: Optional[str] = None
+    unhashed_description: Optional[str] = None
+    expiry: Optional[int] = None
+    lnurl_callback: Optional[str] = None
+    lnurl_balance_check: Optional[str] = None
+    extra: Optional[dict] = None
+    webhook: Optional[str] = None
+    bolt11: Optional[str] = None
+
+
+class CreateTopup(BaseModel):
+    id: str
+    amount: int
+
+
+class CreateLnurlAuth(BaseModel):
+    callback: str

lnbits/core/services.py

@@ -21,6 +21,7 @@ from lnbits.settings import (
     send_admin_user_to_saas,
     settings,
 )
+from lnbits.utils.exchange_rates import fiat_amount_as_satoshis, satoshis_amount_as_fiat
 from lnbits.wallets import FAKE_WALLET, get_wallet_class, set_wallet_class
 from lnbits.wallets.base import PaymentResponse, PaymentStatus
 
@@ -44,7 +45,7 @@ from .crud import (
     update_super_user,
 )
 from .helpers import to_valid_user_id
-from .models import Payment
+from .models import Payment, Wallet
 
 
 class PaymentFailure(Exception):
@@ -55,10 +56,47 @@ class InvoiceFailure(Exception):
     pass
 
 
+async def calculate_fiat_amounts(
+    amount: float,
+    wallet_id: str,
+    currency: Optional[str] = None,
+    extra: Optional[Dict] = None,
+    conn: Optional[Connection] = None,
+) -> Tuple[int, Optional[Dict]]:
+    wallet = await get_wallet(wallet_id, conn=conn)
+    assert wallet, "invalid wallet_id"
+    wallet_currency = wallet.currency or settings.lnbits_default_accounting_currency
+
+    if currency and currency != "sat":
+        amount_sat = await fiat_amount_as_satoshis(amount, currency)
+        extra = extra or {}
+        if currency != wallet_currency:
+            extra["fiat_currency"] = currency
+            extra["fiat_amount"] = round(amount, ndigits=3)
+            extra["fiat_rate"] = amount_sat / amount
+    else:
+        amount_sat = int(amount)
+
+    if wallet_currency:
+        if wallet_currency == currency:
+            fiat_amount = amount
+        else:
+            fiat_amount = await satoshis_amount_as_fiat(amount_sat, wallet_currency)
+        extra = extra or {}
+        extra["wallet_fiat_currency"] = wallet_currency
+        extra["wallet_fiat_amount"] = round(fiat_amount, ndigits=3)
+        extra["wallet_fiat_rate"] = amount_sat / fiat_amount
+
+    logger.debug(f"Calculated fiat amounts for {wallet}: {extra=}")
+
+    return amount_sat, extra
+
+
 async def create_invoice(
     *,
     wallet_id: str,
-    amount: int,  # in satoshis
+    amount: float,
+    currency: Optional[str] = "sat",
     memo: str,
     description_hash: Optional[bytes] = None,
     unhashed_description: Optional[bytes] = None,
@@ -76,8 +114,12 @@ async def create_invoice(
     # use the fake wallet if the invoice is for internal use only
     wallet = FAKE_WALLET if internal else get_wallet_class()
 
+    amount_sat, extra = await calculate_fiat_amounts(
+        amount, wallet_id, currency=currency, extra=extra, conn=conn
+    )
+
     ok, checking_id, payment_request, error_message = await wallet.create_invoice(
-        amount=amount,
+        amount=amount_sat,
         memo=invoice_memo,
         description_hash=description_hash,
         unhashed_description=unhashed_description,
@@ -88,7 +130,7 @@ async def create_invoice(
 
     invoice = bolt11.decode(payment_request)
 
-    amount_msat = amount * 1000
+    amount_msat = 1000 * amount_sat
     await create_payment(
         wallet_id=wallet_id,
         checking_id=checking_id,
@@ -115,16 +157,17 @@ async def pay_invoice(
 ) -> str:
     """
     Pay a Lightning invoice.
-    First, we create a temporary payment in the database with fees set to the reserve fee.
-    We then check whether the balance of the payer would go negative.
-    We then attempt to pay the invoice through the backend.
-    If the payment is successful, we update the payment in the database with the payment details.
+    First, we create a temporary payment in the database with fees set to the reserve
+    fee. We then check whether the balance of the payer would go negative.
+    We then attempt to pay the invoice through the backend. If the payment is
+    successful, we update the payment in the database with the payment details.
     If the payment is unsuccessful, we delete the temporary payment.
-    If the payment is still in flight, we hope that some other process will regularly check for the payment.
+    If the payment is still in flight, we hope that some other process
+    will regularly check for the payment.
     """
     invoice = bolt11.decode(payment_request)
     fee_reserve_msat = fee_reserve(invoice.amount_msat)
-    async with (db.reuse_conn(conn) if conn else db.connect()) as conn:
+    async with db.reuse_conn(conn) if conn else db.connect() as conn:
         temp_id = invoice.payment_hash
         internal_id = f"internal_{invoice.payment_hash}"
 
@@ -133,6 +176,10 @@ async def pay_invoice(
         if max_sat and invoice.amount_msat > max_sat * 1000:
             raise ValueError("Amount in invoice is too high.")
 
+        _, extra = await calculate_fiat_amounts(
+            invoice.amount_msat, wallet_id, extra=extra, conn=conn
+        )
+
         # put all parameters that don't change here
         class PaymentKwargs(TypedDict):
             wallet_id: str
@@ -151,11 +198,13 @@ async def pay_invoice(
             extra=extra,
         )
 
-        # we check if an internal invoice exists that has already been paid (not pending anymore)
+        # we check if an internal invoice exists that has already been paid
+        # (not pending anymore)
         if not await check_internal_pending(invoice.payment_hash, conn=conn):
             raise PaymentFailure("Internal invoice already paid.")
 
-        # check_internal() returns the checking_id of the invoice we're waiting for (pending only)
+        # check_internal() returns the checking_id of the invoice we're waiting for
+        # (pending only)
         internal_checking_id = await check_internal(invoice.payment_hash, conn=conn)
         if internal_checking_id:
             # perform additional checks on the internal payment
@@ -172,7 +221,7 @@ async def pay_invoice(
 
             logger.debug(f"creating temporary internal payment with id {internal_id}")
             # create a new payment from this wallet
-            await create_payment(
+            new_payment = await create_payment(
                 checking_id=internal_id,
                 fee=0,
                 pending=False,
@@ -184,7 +233,7 @@ async def pay_invoice(
             # create a temporary payment here so we can check if
             # the balance is enough in the next step
             try:
-                await create_payment(
+                new_payment = await create_payment(
                     checking_id=temp_id,
                     fee=-fee_reserve_msat,
                     conn=conn,
@@ -202,7 +251,8 @@ async def pay_invoice(
             logger.debug("balance is too low, deleting temporary payment")
             if not internal_checking_id and wallet.balance_msat > -fee_reserve_msat:
                 raise PaymentFailure(
-                    f"You must reserve at least ({round(fee_reserve_msat/1000)} sat) to cover potential routing fees."
+                    f"You must reserve at least ({round(fee_reserve_msat/1000)} sat) to"
+                    " cover potential routing fees."
                 )
             raise PermissionError("Insufficient balance.")
 
@@ -215,6 +265,7 @@ async def pay_invoice(
             await update_payment_status(
                 checking_id=internal_checking_id, pending=False, conn=conn
             )
+        await send_payment_notification(wallet, new_payment)
 
         # notify receiver asynchronously
         from lnbits.tasks import internal_invoice_queue
@@ -231,7 +282,8 @@ async def pay_invoice(
 
         if payment.checking_id and payment.checking_id != temp_id:
             logger.warning(
-                f"backend sent unexpected checking_id (expected: {temp_id} got: {payment.checking_id})"
+                f"backend sent unexpected checking_id (expected: {temp_id} got:"
+                f" {payment.checking_id})"
             )
 
         logger.debug(f"backend: pay_invoice finished {temp_id}")
@@ -248,14 +300,11 @@ async def pay_invoice(
                     conn=conn,
                 )
                 wallet = await get_wallet(wallet_id, conn=conn)
-                if wallet:
-                    await websocketUpdater(
-                        wallet_id,
-                        {
-                            "wallet_balance": wallet.balance or None,
-                            "payment": payment._asdict(),
-                        },
-                    )
+                updated = await get_wallet_payment(
+                    wallet_id, payment.checking_id, conn=conn
+                )
+                if wallet and updated:
+                    await send_payment_notification(wallet, updated)
                 logger.debug(f"payment successful {payment.checking_id}")
         elif payment.checking_id is None and payment.ok is False:
             # payment failed
@@ -269,7 +318,8 @@ async def pay_invoice(
             )
         else:
             logger.warning(
-                f"didn't receive checking_id from backend, payment may be stuck in database: {temp_id}"
+                "didn't receive checking_id from backend, payment may be stuck in"
+                f" database: {temp_id}"
             )
 
     return invoice.payment_hash
@@ -301,9 +351,10 @@ async def redeem_lnurl_withdraw(
             extra=extra,
             conn=conn,
         )
-    except:
+    except Exception:
         logger.warning(
-            f"failed to create invoice on redeem_lnurl_withdraw from {lnurl}. params: {res}"
+            f"failed to create invoice on redeem_lnurl_withdraw "
+            f"from {lnurl}. params: {res}"
         )
         return None
 
@@ -422,13 +473,26 @@ async def check_transaction_status(
     return status
 
 
-# WARN: this same value must be used for balance check and passed to WALLET.pay_invoice(), it may cause a vulnerability if the values differ
+# WARN: this same value must be used for balance check and passed to
+# WALLET.pay_invoice(), it may cause a vulnerability if the values differ
 def fee_reserve(amount_msat: int) -> int:
     reserve_min = settings.lnbits_reserve_fee_min
     reserve_percent = settings.lnbits_reserve_fee_percent
     return max(int(reserve_min), int(amount_msat * reserve_percent / 100.0))
 
 
+async def send_payment_notification(wallet: Wallet, payment: Payment):
+    await websocketUpdater(
+        wallet.id,
+        json.dumps(
+            {
+                "wallet_balance": wallet.balance,
+                "payment": payment.dict(),
+            }
+        ),
+    )
+
+
 async def update_wallet_balance(wallet_id: str, amount: int):
     payment_hash, _ = await create_invoice(
         wallet_id=wallet_id,
@@ -485,7 +549,7 @@ def update_cached_settings(sets_dict: dict):
         if key not in readonly_variables:
             try:
                 setattr(settings, key, value)
-            except:
+            except Exception:
                 logger.warning(f"Failed overriding setting: {key}, value: {value}")
     if "super_user" in sets_dict:
         setattr(settings, "super_user", sets_dict["super_user"])
@@ -506,10 +570,11 @@ async def init_admin_settings(super_user: Optional[str] = None) -> SuperSettings
 
 
 class WebsocketConnectionManager:
-    def __init__(self):
+    def __init__(self) -> None:
         self.active_connections: List[WebSocket] = []
 
-    async def connect(self, websocket: WebSocket):
+    async def connect(self, websocket: WebSocket, item_id: str):
+        logger.debug(f"Websocket connected to {item_id}")
         await websocket.accept()
         self.active_connections.append(websocket)
 

lnbits/core/static/js/service-worker.js

@@ -1,6 +1,6 @@
 // update cache version every time there is a new deployment
 // so the service worker reinitializes the cache
-const CACHE_VERSION = 40
+const CACHE_VERSION = 53
 const CURRENT_CACHE = `lnbits-${CACHE_VERSION}-`
 
 const getApiKey = request => {

lnbits/core/static/js/wallet.js

@@ -137,6 +137,9 @@ new Vue({
           comment: ''
         },
         paymentChecker: null,
+        copy: {
+          show: false
+        },
         camera: {
           show: false,
           camera: 'auto'
@@ -172,6 +175,12 @@ new Vue({
       },
       paymentsCSV: {
         columns: [
+          {
+            name: 'pending',
+            align: 'left',
+            label: 'Pending',
+            field: 'pending'
+          },
           {
             name: 'memo',
             align: 'left',
@@ -221,6 +230,18 @@ new Vue({
             align: 'right',
             label: this.$t('webhook'),
             field: 'webhook'
+          },
+          {
+            name: 'fiat_currency',
+            align: 'right',
+            label: 'Fiat Currency',
+            field: row => row.extra.wallet_fiat_currency
+          },
+          {
+            name: 'fiat_amount',
+            align: 'right',
+            label: 'Fiat Amount',
+            field: row => row.extra.wallet_fiat_amount
           }
         ],
         filter: null,
@@ -235,7 +256,10 @@ new Vue({
       },
       balance: 0,
       credit: 0,
-      newName: ''
+      update: {
+        name: null,
+        currency: null
+      }
     }
   },
   computed: {
@@ -292,7 +316,6 @@ new Vue({
       this.receive.data.amount = null
       this.receive.data.memo = null
       this.receive.unit = 'sat'
-      this.receive.paymentChecker = null
       this.receive.minMax = [0, 2100000000000000]
       this.receive.lnurl = null
       this.focusInput('setAmount')
@@ -302,6 +325,8 @@ new Vue({
       this.parse.invoice = null
       this.parse.lnurlpay = null
       this.parse.lnurlauth = null
+      this.parse.copy.show =
+        window.isSecureContext && navigator.clipboard?.readText !== undefined
       this.parse.data.request = ''
       this.parse.data.comment = ''
       this.parse.data.paymentChecker = null
@@ -334,11 +359,6 @@ new Vue({
           LNbits.utils.notifyApiError(error)
         })
     },
-    closeReceiveDialog: function () {
-      setTimeout(() => {
-        clearInterval(this.receive.paymentChecker)
-      }, 10000)
-    },
     closeParseDialog: function () {
       setTimeout(() => {
         clearInterval(this.parse.paymentChecker)
@@ -351,7 +371,6 @@ new Vue({
       if (this.receive.paymentHash === paymentHash) {
         this.receive.show = false
         this.receive.paymentHash = null
-        clearInterval(this.receive.paymentChecker)
       }
     },
     createInvoice: function () {
@@ -396,19 +415,7 @@ new Vue({
             }
           }
 
-          clearInterval(this.receive.paymentChecker)
-          setTimeout(() => {
-            clearInterval(this.receive.paymentChecker)
-          }, 40000)
-          this.receive.paymentChecker = setInterval(() => {
-            let hash = response.data.payment_hash
-
-            LNbits.api.getPayment(this.g.wallet, hash).then(response => {
-              if (response.data.paid) {
-                this.onPaymentReceived(hash)
-              }
-            })
-          }, 5000)
+          this.fetchPayments()
         })
         .catch(err => {
           LNbits.utils.notifyApiError(err)
@@ -713,16 +720,12 @@ new Vue({
           }
         })
     },
-    updateWalletName: function () {
-      let newName = this.newName
-      let adminkey = this.g.wallet.adminkey
-      if (!newName || !newName.length) return
+    updateWallet: function (data) {
       LNbits.api
-        .request('PUT', '/api/v1/wallet/' + newName, adminkey, {})
+        .request('PATCH', '/api/v1/wallet', this.g.wallet.adminkey, data)
         .then(res => {
-          this.newName = ''
           this.$q.notify({
-            message: `Wallet named updated.`,
+            message: `Wallet updated.`,
             type: 'positive',
             timeout: 3500
           })
@@ -733,7 +736,6 @@ new Vue({
           )
         })
         .catch(err => {
-          this.newName = ''
           LNbits.utils.notifyApiError(err)
         })
     },
@@ -787,14 +789,15 @@ new Vue({
       // status is important for export but it is not in paymentsTable
       // because it is manually added with payment detail link and icons
       // and would cause duplication in the list
-      let columns = structuredClone(this.paymentsCSV.columns)
-      columns.unshift({
-        name: 'pending',
-        align: 'left',
-        label: 'Pending',
-        field: 'pending'
+      LNbits.api.getPayments(this.g.wallet, {}).then(response => {
+        const payments = response.data.data.map(LNbits.map.payment)
+        LNbits.utils.exportCSV(this.paymentsCSV.columns, payments)
+      })
+    },
+    pasteToTextArea: function () {
+      navigator.clipboard.readText().then(text => {
+        this.$refs.textArea.value = text
       })
-      LNbits.utils.exportCSV(columns, this.payments)
     }
   },
   watch: {
@@ -806,6 +809,9 @@ new Vue({
     this.fetchBalance()
     this.fetchPayments()
 
+    this.update.name = this.g.wallet.name
+    this.update.currency = this.g.wallet.currency
+
     LNbits.api
       .request('GET', '/api/v1/currencies')
       .then(response => {

lnbits/core/tasks.py

@@ -1,43 +1,34 @@
 import asyncio
-from typing import Dict, Optional
+from typing import Dict
 
 import httpx
 from loguru import logger
 
 from lnbits.settings import get_wallet_class, settings
-from lnbits.tasks import SseListenersDict, register_invoice_listener
+from lnbits.tasks import (
+    SseListenersDict,
+    create_permanent_task,
+    create_task,
+    register_invoice_listener,
+)
 
 from . import db
 from .crud import get_balance_notify, get_wallet
 from .models import Payment
-from .services import get_balance_delta, switch_to_voidwallet, websocketUpdater
+from .services import get_balance_delta, send_payment_notification, switch_to_voidwallet
 
 api_invoice_listeners: Dict[str, asyncio.Queue] = SseListenersDict(
     "api_invoice_listeners"
 )
 
-killswitch: Optional[asyncio.Task] = None
-watchdog: Optional[asyncio.Task] = None
 
-
-async def register_killswitch():
+def register_killswitch():
     """
-    Registers a killswitch which will check lnbits-status repository
-    for a signal from LNbits and will switch to VoidWallet if the killswitch is triggered.
+    Registers a killswitch which will check lnbits-status repository for a signal from
+    LNbits and will switch to VoidWallet if the killswitch is triggered.
     """
     logger.debug("Starting killswitch task")
-    global killswitch
-    killswitch = asyncio.create_task(killswitch_task())
-
-
-async def unregister_killswitch():
-    """
-    Unregisters a killswitch taskl
-    """
-    global killswitch
-    if killswitch:
-        logger.debug("Stopping killswitch task")
-        killswitch.cancel()
+    create_permanent_task(killswitch_task)
 
 
 async def killswitch_task():
@@ -57,7 +48,8 @@ async def killswitch_task():
                             await switch_to_voidwallet()
                 except (httpx.ConnectError, httpx.RequestError):
                     logger.error(
-                        f"Cannot fetch lnbits status manifest. {settings.lnbits_status_manifest}"
+                        "Cannot fetch lnbits status manifest."
+                        f" {settings.lnbits_status_manifest}"
                     )
         await asyncio.sleep(settings.lnbits_killswitch_interval * 60)
 
@@ -67,20 +59,9 @@ async def register_watchdog():
     Registers a watchdog which will check lnbits balance and nodebalance
     and will switch to VoidWallet if the watchdog delta is reached.
     """
-    # TODO: implement watchdog porperly
+    # TODO: implement watchdog properly
     # logger.debug("Starting watchdog task")
-    # global watchdog
-    # watchdog = asyncio.create_task(watchdog_task())
-
-
-async def unregister_watchdog():
-    """
-    Unregisters a watchdog task
-    """
-    global watchdog
-    if watchdog:
-        logger.debug("Stopping watchdog task")
-        watchdog.cancel()
+    # create_permanent_task(watchdog_task)
 
 
 async def watchdog_task():
@@ -98,22 +79,23 @@ async def watchdog_task():
         await asyncio.sleep(settings.lnbits_watchdog_interval * 60)
 
 
-async def register_task_listeners():
+def register_task_listeners():
     """
-    Registers an invoice listener queue for the core tasks.
-    Incoming payaments in this queue will eventually trigger the signals sent to all other extensions
+    Registers an invoice listener queue for the core tasks. Incoming payments in this
+    queue will eventually trigger the signals sent to all other extensions
     and fulfill other core tasks such as dispatching webhooks.
     """
     invoice_paid_queue = asyncio.Queue(5)
     # we register invoice_paid_queue to receive all incoming invoices
     register_invoice_listener(invoice_paid_queue, "core/tasks.py")
     # register a worker that will react to invoices
-    asyncio.create_task(wait_for_paid_invoices(invoice_paid_queue))
+    create_task(wait_for_paid_invoices(invoice_paid_queue))
 
 
 async def wait_for_paid_invoices(invoice_paid_queue: asyncio.Queue):
     """
-    This worker dispatches events to all extensions, dispatches webhooks and balance notifys.
+    This worker dispatches events to all extensions,
+    dispatches webhooks and balance notifys.
     """
     while True:
         payment = await invoice_paid_queue.get()
@@ -122,13 +104,7 @@ async def wait_for_paid_invoices(invoice_paid_queue: asyncio.Queue):
         await dispatch_api_invoice_listeners(payment)
         wallet = await get_wallet(payment.wallet_id)
         if wallet:
-            await websocketUpdater(
-                payment.wallet_id,
-                {
-                    "wallet_balance": wallet.balance or None,
-                    "payment": payment.dict(),
-                },
-            )
+            await send_payment_notification(wallet, payment)
         # dispatch webhook
         if payment.webhook and not payment.webhook_status:
             await dispatch_webhook(payment)
@@ -161,11 +137,15 @@ async def dispatch_webhook(payment: Payment):
     """
     Dispatches the webhook to the webhook url.
     """
+    logger.debug("sending webhook", payment.webhook)
+
+    if not payment.webhook:
+        return await mark_webhook_sent(payment, -1)
+
     async with httpx.AsyncClient() as client:
         data = payment.dict()
         try:
-            logger.debug("sending webhook", payment.webhook)
-            r = await client.post(payment.webhook, json=data, timeout=40)  # type: ignore
+            r = await client.post(payment.webhook, json=data, timeout=40)
             await mark_webhook_sent(payment, r.status_code)
         except (httpx.ConnectError, httpx.RequestError):
             await mark_webhook_sent(payment, -1)

lnbits/core/templates/admin/_tab_funding.html

@@ -47,7 +47,8 @@
               <p>Invoice Expiry</p>
               <q-input
                 filled
-                v-model="formData.lightning_invoice_expiry"
+                v-model.number="formData.lightning_invoice_expiry"
+                type="number"
                 label="Invoice expiry (seconds)"
                 mask="#######"
               >

lnbits/core/templates/admin/_tab_server.html

@@ -67,6 +67,18 @@
           ></q-select>
           <br />
         </div>
+        <div class="col-12 col-md-6">
+          <p>Default Accounting Currency</p>
+          <q-select
+            filled
+            v-model="formData.lnbits_default_accounting_currency"
+            clearable
+            hint="Default currency for accounting"
+            label="Currency"
+            :options="formData.lnbits_allowed_currencies.length ? formData.lnbits_allowed_currencies : {{ currencies }}"
+          ></q-select>
+          <br />
+        </div>
       </div>
       <div class="row q-col-gutter-md">
         <div class="col-12 col-md-6">

lnbits/core/templates/admin/index.html

@@ -236,6 +236,23 @@
               }
             }
           ],
+          [
+            'CoreLightningRestWallet',
+            {
+              corelightning_rest_url: {
+                value: null,
+                label: 'Endpoint'
+              },
+              corelightning_rest_cert: {
+                value: null,
+                label: 'Certificate'
+              },
+              corelightning_rest_macaroon: {
+                value: null,
+                label: 'Macaroon'
+              }
+            }
+          ],
           [
             'LndRestWallet',
             {
@@ -376,7 +393,7 @@
         ])
       }
     },
-    created: function () {
+    created() {
       this.getSettings()
       this.getAudit()
       this.balance = +'{{ balance|safe }}'
@@ -593,7 +610,7 @@
           .then(response => {
             this.isSuperUser = response.data.is_super_user || false
             this.settings = response.data
-            this.formData = _.clone(this.settings)
+            this.formData = {...this.settings}
             this.updateFundingData()
             this.getNotifications()
           })
@@ -624,7 +641,9 @@
             this.updateFundingData()
             this.$q.notify({
               type: 'positive',
-              message: 'Success! Settings changed!',
+              message: `Success! Settings changed! ${
+                this.needsRestart ? 'Restart required!' : ''
+              }`,
               icon: null
             })
           })

lnbits/core/templates/core/index.html

@@ -124,6 +124,16 @@
             </a>
           </div>
           <div class="col q-pl-md">
+            <a href="https://start9.com/">
+              <q-img
+                contain
+                :src="($q.dark.isActive) ? '/static/images/start9.png' : '/static/images/start9l.png'"
+              ></q-img>
+            </a>
+          </div>
+        </div>
+        <div class="row">
+          <div class="col">
             <a href="https://getumbrel.com/">
               <q-img
                 contain
@@ -131,9 +141,7 @@
               ></q-img>
             </a>
           </div>
-        </div>
-        <div class="row">
-          <div class="col">
+          <div class="col q-pl-md">
             <a href="https://mynodebtc.com">
               <q-img
                 contain
@@ -141,7 +149,9 @@
               ></q-img>
             </a>
           </div>
-          <div class="col q-pl-md">
+        </div>
+        <div class="row">
+          <div class="col">
             <a href="https://github.com/shesek/spark-wallet">
               <q-img
                 contain
@@ -149,9 +159,7 @@
               ></q-img>
             </a>
           </div>
-        </div>
-        <div class="row">
-          <div class="col">
+          <div class="col q-pl-md">
             <a href="https://voltage.cloud">
               <q-img
                 contain
@@ -159,7 +167,6 @@
               ></q-img>
             </a>
           </div>
-          <div class="col q-pl-md"></div>
         </div>
       </div>
 

lnbits/core/templates/core/wallet.html

@@ -211,10 +211,17 @@
                     parseFloat(String(props.row.fsat).replaceAll(",", "")) / 100
                     }}
                   </q-td>
-
                   <q-td auto-width key="amount" v-else :props="props">
                     {{ props.row.fsat }}<br />
-                    <i>fee {{ props.row.fee/1000 }}</i>
+                    <i v-if="props.row.extra.wallet_fiat_currency">
+                      {{ props.row.extra.wallet_fiat_currency }} {{
+                      props.row.extra.wallet_fiat_amount }}
+                      <br />
+                    </i>
+                    <i v-if="props.row.extra.fiat_currency">
+                      {{ props.row.extra.fiat_currency }} {{
+                      props.row.extra.fiat_amount }}
+                    </i>
                   </q-td>
                 </q-tr>
 
@@ -361,23 +368,54 @@
                       <div class="" style="max-width: 320px">
                         <q-input
                           filled
-                          v-model.trim="newName"
-                          label="Label"
-                          dense="dense"
+                          v-model.trim="update.name"
+                          label="Name"
+                          dense
                           @update:model-value="(e) => console.log(e)"
                         />
                       </div>
                       <q-btn
-                        :disable="!newName.length"
+                        :disable="!update.name.length"
                         unelevated
                         class="q-mt-sm"
                         color="primary"
                         :label="$t('update_name')"
-                        @click="updateWalletName()"
+                        @click="updateWallet({ name: update.name })"
                       ></q-btn>
                     </q-card-section>
                   </q-card>
                 </q-expansion-item>
+                <q-separator></q-separator>
+                <q-expansion-item
+                  group="extras"
+                  icon="attach_money"
+                  :label="$t('fiat_tracking')"
+                >
+                  <q-card>
+                    <q-card-section>
+                      <div style="max-width: 360px">
+                        <q-select
+                          filled
+                          dense
+                          clearable
+                          v-model="update.currency"
+                          type="text"
+                          :label="$t('currency')"
+                          :options="receive.units.filter((u) => u !== 'sat')"
+                        ></q-select>
+                      </div>
+                      <q-btn
+                        :disable="!update.name.length"
+                        unelevated
+                        class="q-mt-sm"
+                        color="primary"
+                        :label="$t('update_currency')"
+                        @click="updateWallet({ currency: update.currency || '' })"
+                      ></q-btn>
+                    </q-card-section>
+                  </q-card>
+                </q-expansion-item>
+
                 <q-separator></q-separator>
                 <q-expansion-item
                   group="extras"
@@ -428,7 +466,7 @@
         </div>
       </div>
 
-      <q-dialog v-model="receive.show" @hide="closeReceiveDialog">
+      <q-dialog v-model="receive.show">
         {% raw %}
         <q-card
           v-if="!receive.paymentReq"
@@ -711,6 +749,7 @@
                 v-model.trim="parse.data.request"
                 type="textarea"
                 :label="$t('paste_invoice_label')"
+                ref="textArea"
               >
               </q-input>
               <div class="row q-mt-lg">
@@ -721,6 +760,17 @@
                   type="submit"
                   :label="$t('read')"
                 ></q-btn>
+                <q-icon
+                  name="content_paste"
+                  color="grey"
+                  class="q-mt-xs q-ml-sm q-mr-auto"
+                  v-if="parse.copy.show"
+                  @click="pasteToTextArea"
+                  ><q-tooltip
+                    >{% raw %}{{$t('paste_from_clipboard')}}{% endraw
+                    %}</q-tooltip
+                  ></q-icon
+                >
                 <q-btn
                   v-close-popup
                   flat

lnbits/core/views/admin_api.py

@@ -6,12 +6,12 @@ from subprocess import Popen
 from typing import Optional
 from urllib.parse import urlparse
 
-from fastapi import Body, Depends
+from fastapi import Depends
 from fastapi.responses import FileResponse
 from starlette.exceptions import HTTPException
 
 from lnbits.core.crud import get_wallet
-from lnbits.core.models import User
+from lnbits.core.models import CreateTopup, User
 from lnbits.core.services import (
     get_balance_delta,
     update_cached_settings,
@@ -25,7 +25,12 @@ from .. import core_app, core_app_extra
 from ..crud import delete_admin_settings, get_admin_settings, update_admin_settings
 
 
-@core_app.get("/admin/api/v1/audit", dependencies=[Depends(check_admin)])
+@core_app.get(
+    "/admin/api/v1/audit",
+    name="Audit",
+    description="show the current balance of the node and the LNbits database",
+    dependencies=[Depends(check_admin)],
+)
 async def api_auditor():
     try:
         delta, node_balance, total_balance = await get_balance_delta()
@@ -34,7 +39,7 @@ async def api_auditor():
             "node_balance_msats": int(node_balance),
             "lnbits_balance_msats": int(total_balance),
         }
-    except:
+    except Exception:
         raise HTTPException(
             status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
             detail="Could not audit balance.",
@@ -86,15 +91,14 @@ async def api_restart_server() -> dict[str, str]:
 
 @core_app.put(
     "/admin/api/v1/topup/",
+    name="Topup",
     status_code=HTTPStatus.OK,
     dependencies=[Depends(check_super_user)],
 )
-async def api_topup_balance(
-    id: str = Body(...), amount: int = Body(...)
-) -> dict[str, str]:
+async def api_topup_balance(data: CreateTopup) -> dict[str, str]:
     try:
-        await get_wallet(id)
-    except:
+        await get_wallet(data.id)
+    except Exception:
         raise HTTPException(
             status_code=HTTPStatus.FORBIDDEN, detail="wallet does not exist."
         )
@@ -104,7 +108,7 @@ async def api_topup_balance(
             status_code=HTTPStatus.FORBIDDEN, detail="VoidWallet active"
         )
 
-    await update_wallet_balance(wallet_id=id, amount=int(amount))
+    await update_wallet_balance(wallet_id=data.id, amount=int(data.amount))
 
     return {"status": "Success"}
 
@@ -126,10 +130,10 @@ async def api_download_backup() -> FileResponse:
         p = urlparse(db_url)
         command = (
             f"pg_dump --host={p.hostname} "
-            f'--dbname={p.path.replace("/", "")} '
+            f"--dbname={p.path.replace('/', '')} "
             f"--username={p.username} "
-            f"--no-password "
-            f"--format=c "
+            "--no-password "
+            "--format=c "
             f"--file={pg_backup_filename}"
         )
         proc = Popen(

lnbits/core/views/api.py

@@ -4,17 +4,15 @@ import json
 import uuid
 from http import HTTPStatus
 from io import BytesIO
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Union
 from urllib.parse import ParseResult, parse_qs, urlencode, urlparse, urlunparse
 
-import async_timeout
 import httpx
 import pyqrcode
 from fastapi import (
     Body,
     Depends,
     Header,
-    Query,
     Request,
     Response,
     WebSocket,
@@ -22,8 +20,6 @@ from fastapi import (
 )
 from fastapi.exceptions import HTTPException
 from loguru import logger
-from pydantic import BaseModel
-from pydantic.fields import Field
 from sse_starlette.sse import EventSourceResponse
 from starlette.responses import RedirectResponse, StreamingResponse
 
@@ -32,7 +28,18 @@ from lnbits.core.helpers import (
     migrate_extension_database,
     stop_extension_background_work,
 )
-from lnbits.core.models import Payment, PaymentFilters, User, Wallet
+from lnbits.core.models import (
+    ConversionData,
+    CreateInvoice,
+    CreateLnurl,
+    CreateLnurlAuth,
+    DecodePayment,
+    Payment,
+    PaymentFilters,
+    User,
+    Wallet,
+    WalletType,
+)
 from lnbits.db import Filters, Page
 from lnbits.decorators import (
     WalletTypeInfo,
@@ -96,7 +103,7 @@ async def health():
 
 @core_app.get("/api/v1/wallet")
 async def api_wallet(wallet: WalletTypeInfo = Depends(get_key_type)):
-    if wallet.wallet_type == 0:
+    if wallet.wallet_type == WalletType.admin:
         return {
             "id": wallet.wallet.id,
             "name": wallet.wallet.name,
@@ -107,7 +114,7 @@ async def api_wallet(wallet: WalletTypeInfo = Depends(get_key_type)):
 
 
 @core_app.put("/api/v1/wallet/{new_name}")
-async def api_update_wallet(
+async def api_update_wallet_name(
     new_name: str, wallet: WalletTypeInfo = Depends(require_admin_key)
 ):
     await update_wallet(wallet.wallet.id, new_name)
@@ -118,6 +125,15 @@ async def api_update_wallet(
     }
 
 
+@core_app.patch("/api/v1/wallet", response_model=Wallet)
+async def api_update_wallet(
+    name: Optional[str] = Body(None),
+    currency: Optional[str] = Body(None),
+    wallet: WalletTypeInfo = Depends(require_admin_key),
+):
+    return await update_wallet(wallet.wallet.id, name, currency)
+
+
 @core_app.get(
     "/api/v1/payments",
     name="Payment List",
@@ -179,57 +195,37 @@ async def api_payments_paginated(
     return page
 
 
-class CreateInvoiceData(BaseModel):
-    out: Optional[bool] = True
-    amount: float = Query(None, ge=0)
-    memo: Optional[str] = None
-    unit: Optional[str] = "sat"
-    description_hash: Optional[str] = None
-    unhashed_description: Optional[str] = None
-    expiry: Optional[int] = None
-    lnurl_callback: Optional[str] = None
-    lnurl_balance_check: Optional[str] = None
-    extra: Optional[dict] = None
-    webhook: Optional[str] = None
-    internal: Optional[bool] = False
-    bolt11: Optional[str] = None
-
-
-async def api_payments_create_invoice(data: CreateInvoiceData, wallet: Wallet):
+async def api_payments_create_invoice(data: CreateInvoice, wallet: Wallet):
+    description_hash = b""
+    unhashed_description = b""
+    memo = data.memo or settings.lnbits_site_title
     if data.description_hash or data.unhashed_description:
-        try:
-            description_hash = (
-                bytes.fromhex(data.description_hash) if data.description_hash else b""
-            )
-            unhashed_description = (
-                bytes.fromhex(data.unhashed_description)
-                if data.unhashed_description
-                else b""
-            )
-        except ValueError:
-            raise HTTPException(
-                status_code=HTTPStatus.BAD_REQUEST,
-                detail="'description_hash' and 'unhashed_description' must be a valid hex strings",
-            )
+        if data.description_hash:
+            try:
+                description_hash = bytes.fromhex(data.description_hash)
+            except ValueError:
+                raise HTTPException(
+                    status_code=HTTPStatus.BAD_REQUEST,
+                    detail="'description_hash' must be a valid hex string",
+                )
+        if data.unhashed_description:
+            try:
+                unhashed_description = bytes.fromhex(data.unhashed_description)
+            except ValueError:
+                raise HTTPException(
+                    status_code=HTTPStatus.BAD_REQUEST,
+                    detail="'unhashed_description' must be a valid hex string",
+                )
+        # do not save memo if description_hash or unhashed_description is set
         memo = ""
-    else:
-        description_hash = b""
-        unhashed_description = b""
-        memo = data.memo or settings.lnbits_site_title
-
-    if data.unit == "sat":
-        amount = int(data.amount)
-    else:
-        assert data.unit is not None, "unit not set"
-        price_in_sats = await fiat_amount_as_satoshis(data.amount, data.unit)
-        amount = price_in_sats
 
     async with db.connect() as conn:
         try:
-            _, payment_request = await create_invoice(
+            payment_hash, payment_request = await create_invoice(
                 wallet_id=wallet.id,
-                amount=amount,
+                amount=data.amount,
                 memo=memo,
+                currency=data.unit,
                 description_hash=description_hash,
                 unhashed_description=unhashed_description,
                 expiry=data.expiry,
@@ -238,6 +234,12 @@ async def api_payments_create_invoice(data: CreateInvoiceData, wallet: Wallet):
                 internal=data.internal,
                 conn=conn,
             )
+            # NOTE: we get the checking_id with a seperate query because create_invoice
+            # does not return it and it would be a big hustle to change its return type
+            # (used across extensions)
+            payment_db = await get_standalone_payment(payment_hash, conn=conn)
+            assert payment_db is not None, "payment not found"
+            checking_id = payment_db.checking_id
         except InvoiceFailure as e:
             raise HTTPException(status_code=520, detail=str(e))
         except Exception as exc:
@@ -280,7 +282,7 @@ async def api_payments_create_invoice(data: CreateInvoiceData, wallet: Wallet):
         "payment_hash": invoice.payment_hash,
         "payment_request": payment_request,
         # maintain backwards compatibility with API clients:
-        "checking_id": invoice.payment_hash,
+        "checking_id": checking_id,
         "lnurl_response": lnurl_response,
     }
 
@@ -306,13 +308,22 @@ async def api_payments_pay_invoice(bolt11: str, wallet: Wallet):
 
 @core_app.post(
     "/api/v1/payments",
+    summary="Create or pay an invoice",
+    description="""
+        This endpoint can be used both to generate and pay a BOLT11 invoice.
+        To generate a new invoice for receiving funds into the authorized account,
+        specify at least the first four fields in the POST body: `out: false`,
+        `amount`, `unit`, and `memo`. To pay an arbitrary invoice from the funds
+        already in the authorized account, specify `out: true` and use the `bolt11`
+        field to supply the BOLT11 invoice to be paid.
+    """,
     status_code=HTTPStatus.CREATED,
 )
 async def api_payments_create(
     wallet: WalletTypeInfo = Depends(require_invoice_key),
-    invoiceData: CreateInvoiceData = Body(...),
+    invoiceData: CreateInvoice = Body(...),
 ):
-    if invoiceData.out is True and wallet.wallet_type == 0:
+    if invoiceData.out is True and wallet.wallet_type == WalletType.admin:
         if not invoiceData.bolt11:
             raise HTTPException(
                 status_code=HTTPStatus.BAD_REQUEST,
@@ -331,17 +342,9 @@ async def api_payments_create(
         )
 
 
-class CreateLNURLData(BaseModel):
-    description_hash: str
-    callback: str
-    amount: int
-    comment: Optional[str] = None
-    description: Optional[str] = None
-
-
 @core_app.post("/api/v1/payments/lnurl")
 async def api_payments_pay_lnurl(
-    data: CreateLNURLData, wallet: WalletTypeInfo = Depends(require_admin_key)
+    data: CreateLnurl, wallet: WalletTypeInfo = Depends(require_admin_key)
 ):
     domain = urlparse(data.callback).netloc
 
@@ -377,13 +380,23 @@ async def api_payments_pay_lnurl(
     if invoice.amount_msat != data.amount:
         raise HTTPException(
             status_code=HTTPStatus.BAD_REQUEST,
-            detail=f"{domain} returned an invalid invoice. Expected {data.amount} msat, got {invoice.amount_msat}.",
+            detail=(
+                (
+                    f"{domain} returned an invalid invoice. Expected"
+                    f" {data.amount} msat, got {invoice.amount_msat}."
+                ),
+            ),
         )
 
     if invoice.description_hash != data.description_hash:
         raise HTTPException(
             status_code=HTTPStatus.BAD_REQUEST,
-            detail=f"{domain} returned an invalid invoice. Expected description_hash == {data.description_hash}, got {invoice.description_hash}.",
+            detail=(
+                (
+                    f"{domain} returned an invalid invoice. Expected description_hash"
+                    f" == {data.description_hash}, got {invoice.description_hash}."
+                ),
+            ),
         )
 
     extra = {}
@@ -421,34 +434,18 @@ async def subscribe_wallet_invoices(request: Request, wallet: Wallet):
     logger.debug(f"adding sse listener for wallet: {uid}")
     api_invoice_listeners[uid] = payment_queue
 
-    send_queue: asyncio.Queue[Tuple[str, Payment]] = asyncio.Queue(0)
-
-    async def payment_received() -> None:
-        while True:
-            try:
-                async with async_timeout.timeout(1):
-                    payment: Payment = await payment_queue.get()
-                    if payment.wallet_id == this_wallet_id:
-                        logger.debug("sse listener: payment received", payment)
-                        await send_queue.put(("payment-received", payment))
-            except asyncio.TimeoutError:
-                pass
-
-    task = asyncio.create_task(payment_received())
-
     try:
         while True:
             if await request.is_disconnected():
                 await request.close()
                 break
-            typ, data = await send_queue.get()
-            if data:
-                jdata = json.dumps(dict(data.dict(), pending=False))
-                yield dict(data=jdata, event=typ)
+            payment: Payment = await payment_queue.get()
+            if payment.wallet_id == this_wallet_id:
+                logger.debug("sse listener: payment received", payment)
+                yield dict(data=payment.json(), event="payment-received")
     except asyncio.CancelledError:
         logger.debug(f"removing listener for wallet {uid}")
         api_invoice_listeners.pop(uid)
-        task.cancel()
         return
 
 
@@ -468,10 +465,11 @@ async def api_payments_sse(
 async def api_payment(payment_hash, X_Api_Key: Optional[str] = Header(None)):
     # We use X_Api_Key here because we want this call to work with and without keys
     # If a valid key is given, we also return the field "details", otherwise not
-    wallet = await get_wallet_for_key(X_Api_Key) if type(X_Api_Key) == str else None  # type: ignore
+    wallet = await get_wallet_for_key(X_Api_Key) if isinstance(X_Api_Key, str) else None
 
-    # we have to specify the wallet id here, because postgres and sqlite return internal payments in different order
-    # and get_standalone_payment otherwise just fetches the first one, causing unpredictable results
+    # we have to specify the wallet id here, because postgres and sqlite return
+    # internal payments in different order and get_standalone_payment otherwise
+    # just fetches the first one, causing unpredictable results
     payment = await get_standalone_payment(
         payment_hash, wallet_id=wallet.id if wallet else None
     )
@@ -513,7 +511,7 @@ async def api_lnurlscan(code: str, wallet: WalletTypeInfo = Depends(get_key_type
     try:
         url = lnurl.decode(code)
         domain = urlparse(url).netloc
-    except:
+    except Exception:
         # parse internet identifier (user@domain.com)
         name_domain = code.split("@")
         if len(name_domain) == 2 and len(name_domain[1].split(".")) >= 2:
@@ -541,8 +539,9 @@ async def api_lnurlscan(code: str, wallet: WalletTypeInfo = Depends(get_key_type
         assert lnurlauth_key.verifying_key
         params.update(pubkey=lnurlauth_key.verifying_key.to_string("compressed").hex())
     else:
-        async with httpx.AsyncClient() as client:
+        async with httpx.AsyncClient(follow_redirects=True) as client:
             r = await client.get(url, timeout=5)
+            r.raise_for_status()
             if r.is_error:
                 raise HTTPException(
                     status_code=HTTPStatus.SERVICE_UNAVAILABLE,
@@ -622,10 +621,6 @@ async def api_lnurlscan(code: str, wallet: WalletTypeInfo = Depends(get_key_type
     return params
 
 
-class DecodePayment(BaseModel):
-    data: str
-
-
 @core_app.post("/api/v1/payments/decode", status_code=HTTPStatus.OK)
 async def api_payments_decode(data: DecodePayment, response: Response):
     payment_str = data.data
@@ -647,20 +642,16 @@ async def api_payments_decode(data: DecodePayment, response: Response):
                 "route_hints": invoice.route_hints,
                 "min_final_cltv_expiry": invoice.min_final_cltv_expiry,
             }
-    except:
+    except Exception:
         response.status_code = HTTPStatus.BAD_REQUEST
         return {"message": "Failed to decode"}
 
 
-class Callback(BaseModel):
-    callback: str = Query(...)
-
-
 @core_app.post("/api/v1/lnurlauth")
 async def api_perform_lnurlauth(
-    callback: Callback, wallet: WalletTypeInfo = Depends(require_admin_key)
+    data: CreateLnurlAuth, wallet: WalletTypeInfo = Depends(require_admin_key)
 ):
-    err = await perform_lnurlauth(callback.callback, wallet=wallet)
+    err = await perform_lnurlauth(data.callback, wallet=wallet)
     if err:
         raise HTTPException(
             status_code=HTTPStatus.SERVICE_UNAVAILABLE, detail=err.reason
@@ -679,12 +670,6 @@ async def api_list_currencies_available():
     return list(currencies.keys())
 
 
-class ConversionData(BaseModel):
-    from_: str = Field("sat", alias="from")
-    amount: float
-    to: str = Query("usd")
-
-
 @core_app.post("/api/v1/conversion")
 async def api_fiat_as_sats(data: ConversionData):
     output = {}
@@ -704,7 +689,7 @@ async def api_fiat_as_sats(data: ConversionData):
 
 
 @core_app.get("/api/v1/qrcode/{data}", response_class=StreamingResponse)
-async def img(request: Request, data):
+async def img(data):
     qr = pyqrcode.create(data)
     stream = BytesIO()
     qr.svg(stream, scale=3)
@@ -724,12 +709,9 @@ async def img(request: Request, data):
     )
 
 
-# UNIVERSAL WEBSOCKET MANAGER
-
-
 @core_app.websocket("/api/v1/ws/{item_id}")
 async def websocket_connect(websocket: WebSocket, item_id: str):
-    await websocketManager.connect(websocket)
+    await websocketManager.connect(websocket, item_id)
     try:
         while True:
             await websocket.receive_text()
@@ -742,7 +724,7 @@ async def websocket_update_post(item_id: str, data: str):
     try:
         await websocketUpdater(item_id, data)
         return {"sent": True, "data": data}
-    except:
+    except Exception:
         return {"sent": False, "data": data}
 
 
@@ -751,7 +733,7 @@ async def websocket_update_get(item_id: str, data: str):
     try:
         await websocketUpdater(item_id, data)
         return {"sent": True, "data": data}
-    except:
+    except Exception:
         return {"sent": False, "data": data}
 
 
@@ -807,7 +789,10 @@ async def api_install_extension(
         ext_info.clean_extension_files()
         raise HTTPException(
             status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
-            detail=f"Failed to install extension {ext_info.id} ({ext_info.installed_version}).",
+            detail=(
+                f"Failed to install extension {ext_info.id} "
+                f"({ext_info.installed_version})."
+            ),
         )
 
 
@@ -830,7 +815,10 @@ async def api_uninstall_extension(ext_id: str, user: User = Depends(check_admin)
         if installed_ext and ext_id in installed_ext.dependencies:
             raise HTTPException(
                 status_code=HTTPStatus.BAD_REQUEST,
-                detail=f"Cannot uninstall. Extension '{installed_ext.name}' depends on this one.",
+                detail=(
+                    f"Cannot uninstall. Extension '{installed_ext.name}' "
+                    "depends on this one."
+                ),
             )
 
     try:
@@ -918,7 +906,11 @@ async def delete_extension_db(ext_id: str):
 # TINYURL
 
 
-@core_app.post("/api/v1/tinyurl")
+@core_app.post(
+    "/api/v1/tinyurl",
+    name="Tinyurl",
+    description="creates a tinyurl",
+)
 async def api_create_tinyurl(
     url: str, endless: bool = False, wallet: WalletTypeInfo = Depends(get_key_type)
 ):
@@ -929,13 +921,17 @@ async def api_create_tinyurl(
                 if tinyurl.wallet == wallet.wallet.inkey:
                     return tinyurl
         return await create_tinyurl(url, endless, wallet.wallet.inkey)
-    except:
+    except Exception:
         raise HTTPException(
             status_code=HTTPStatus.BAD_REQUEST, detail="Unable to create tinyurl"
         )
 
 
-@core_app.get("/api/v1/tinyurl/{tinyurl_id}")
+@core_app.get(
+    "/api/v1/tinyurl/{tinyurl_id}",
+    name="Tinyurl",
+    description="get a tinyurl by id",
+)
 async def api_get_tinyurl(
     tinyurl_id: str, wallet: WalletTypeInfo = Depends(get_key_type)
 ):
@@ -947,13 +943,17 @@ async def api_get_tinyurl(
         raise HTTPException(
             status_code=HTTPStatus.FORBIDDEN, detail="Wrong key provided."
         )
-    except:
+    except Exception:
         raise HTTPException(
             status_code=HTTPStatus.NOT_FOUND, detail="Unable to fetch tinyurl"
         )
 
 
-@core_app.delete("/api/v1/tinyurl/{tinyurl_id}")
+@core_app.delete(
+    "/api/v1/tinyurl/{tinyurl_id}",
+    name="Tinyurl",
+    description="delete a tinyurl by id",
+)
 async def api_delete_tinyurl(
     tinyurl_id: str, wallet: WalletTypeInfo = Depends(get_key_type)
 ):
@@ -966,22 +966,23 @@ async def api_delete_tinyurl(
         raise HTTPException(
             status_code=HTTPStatus.FORBIDDEN, detail="Wrong key provided."
         )
-    except:
+    except Exception:
         raise HTTPException(
             status_code=HTTPStatus.BAD_REQUEST, detail="Unable to delete"
         )
 
 
-@core_app.get("/t/{tinyurl_id}")
+@core_app.get(
+    "/t/{tinyurl_id}",
+    name="Tinyurl",
+    description="redirects a tinyurl by id",
+)
 async def api_tinyurl(tinyurl_id: str):
-    try:
-        tinyurl = await get_tinyurl(tinyurl_id)
-        if tinyurl:
-            response = RedirectResponse(url=tinyurl.url)
-            return response
-        else:
-            return
-    except:
+    tinyurl = await get_tinyurl(tinyurl_id)
+    if tinyurl:
+        response = RedirectResponse(url=tinyurl.url)
+        return response
+    else:
         raise HTTPException(
             status_code=HTTPStatus.NOT_FOUND, detail="unable to find tinyurl"
         )

lnbits/core/views/generic.py

@@ -35,7 +35,9 @@ from ..crud import (
 )
 from ..services import pay_invoice, redeem_lnurl_withdraw
 
-core_html_routes: APIRouter = APIRouter(tags=["Core NON-API Website Routes"])
+core_html_routes: APIRouter = APIRouter(
+    tags=["Core NON-API Website Routes"], include_in_schema=False
+)
 
 
 @core_html_routes.get("/favicon.ico", response_class=FileResponse)
@@ -132,12 +134,12 @@ async def extensions_install(
                     "isAvailable": ext.id in all_extensions,
                     "isAdminOnly": ext.id in settings.lnbits_admin_extensions,
                     "isActive": ext.id not in inactive_extensions,
-                    "latestRelease": dict(ext.latest_release)
-                    if ext.latest_release
-                    else None,
-                    "installedRelease": dict(ext.installed_release)
-                    if ext.installed_release
-                    else None,
+                    "latestRelease": (
+                        dict(ext.latest_release) if ext.latest_release else None
+                    ),
+                    "installedRelease": (
+                        dict(ext.installed_release) if ext.installed_release else None
+                    ),
                 },
                 installable_exts,
             )
@@ -160,13 +162,13 @@ async def extensions_install(
     "/wallet",
     response_class=HTMLResponse,
     description="""
-Args:
-
-just **wallet_name**: create a new user, then create a new wallet for user with wallet_name<br>
-just **user_id**: return the first user wallet or create one if none found (with default wallet_name)<br>
-**user_id** and **wallet_name**: create a new wallet for user with wallet_name<br>
-**user_id** and **wallet_id**: return that wallet if user is the owner<br>
-nothing: create everything<br>
+just **wallet_name**: create a new user, then create a new wallet
+                      for user with wallet_name
+just **user_id**: return the first user wallet or create one if none found
+                  (with default wallet_name)
+**user_id** and **wallet_name**: create a new wallet for user with wallet_name
+**user_id** and **wallet_id**: return that wallet if user is the owner
+nothing: create everything
 """,
 )
 async def wallet(
@@ -210,7 +212,8 @@ async def wallet(
         else:
             wallet = await create_wallet(user_id=user.id, wallet_name=wallet_name)
             logger.info(
-                f"Created new wallet {wallet_name if wallet_name else '(no name)'} for user {user.id}"
+                f"Created new wallet {wallet_name if wallet_name else '(no name)'} for"
+                f" user {user.id}"
             )
 
         return RedirectResponse(
@@ -219,7 +222,9 @@ async def wallet(
         )
 
     logger.debug(
-        f"Access {'user '+ user.id + ' ' if user else ''} {'wallet ' + wallet_name if wallet_name else ''}"
+        "Access "
+        f"{'user '+ user.id + ' ' if user else ''} "
+        f"{'wallet ' + wallet_name if wallet_name else ''}"
     )
     userwallet = user.get_wallet(wallet_id)
     if not userwallet:
@@ -255,7 +260,9 @@ async def lnurl_full_withdraw(request: Request):
         "k1": "0",
         "minWithdrawable": 1000 if wallet.withdrawable_balance else 0,
         "maxWithdrawable": wallet.withdrawable_balance,
-        "defaultDescription": f"{settings.lnbits_site_title} balance withdraw from {wallet.id[0:5]}",
+        "defaultDescription": (
+            f"{settings.lnbits_site_title} balance withdraw from {wallet.id[0:5]}"
+        ),
         "balanceCheck": url_for("/withdraw", external=True, usr=user.id, wal=wallet.id),
     }
 
@@ -275,7 +282,7 @@ async def lnurl_full_withdraw_callback(request: Request):
     async def pay():
         try:
             await pay_invoice(wallet_id=wallet.id, payment_request=pr)
-        except:
+        except Exception:
             pass
 
     asyncio.create_task(pay())
@@ -362,9 +369,11 @@ async def manifest(usr: str):
         "name": settings.lnbits_site_title + " Wallet",
         "icons": [
             {
-                "src": settings.lnbits_custom_logo
-                if settings.lnbits_custom_logo
-                else "https://cdn.jsdelivr.net/gh/lnbits/lnbits@0.3.0/docs/logos/lnbits.png",
+                "src": (
+                    settings.lnbits_custom_logo
+                    if settings.lnbits_custom_logo
+                    else "https://cdn.jsdelivr.net/gh/lnbits/lnbits@0.3.0/docs/logos/lnbits.png"
+                ),
                 "type": "image/png",
                 "sizes": "900x900",
             }

lnbits/core/views/public_api.py

@@ -28,7 +28,7 @@ async def api_public_payment_longpolling(payment_hash):
         expiration = datetime.datetime.fromtimestamp(invoice.date + invoice.expiry)
         if expiration < datetime.datetime.now():
             return {"status": "expired"}
-    except:
+    except Exception:
         raise HTTPException(
             status_code=HTTPStatus.BAD_REQUEST, detail="Invalid bolt11 invoice."
         )

lnbits/db.py

@@ -47,15 +47,6 @@ if settings.lnbits_database_url:
             lambda value, curs: float(value) if value is not None else None,
         )
     )
-    register_type(
-        new_type(
-            (1082, 1083, 1266),
-            "DATE2INT",
-            lambda value, curs: time.mktime(value.timetuple())
-            if value is not None
-            else None,
-        )
-    )
 
     register_type(new_type((1184, 1114), "TIMESTAMP2INT", _parse_timestamp))
 else:
@@ -68,6 +59,15 @@ else:
         )
 
 
+def compat_timestamp_placeholder():
+    if DB_TYPE == POSTGRES:
+        return "to_timestamp(?)"
+    elif DB_TYPE == COCKROACH:
+        return "cast(? AS timestamp)"
+    else:
+        return "?"
+
+
 class Compat:
     type: Optional[str] = "<inherited>"
     schema: Optional[str] = "<inherited>"
@@ -116,15 +116,9 @@ class Compat:
             return "BIGINT"
         return "INT"
 
-    @classmethod
     @property
-    def timestamp_placeholder(cls):
-        if DB_TYPE == POSTGRES:
-            return "to_timestamp(?)"
-        elif DB_TYPE == COCKROACH:
-            return "cast(? AS timestamp)"
-        else:
-            return "?"
+    def timestamp_placeholder(self) -> str:
+        return compat_timestamp_placeholder()
 
 
 class Connection(Compat):
@@ -304,10 +298,10 @@ class Database(Compat):
         yield conn
 
     @classmethod
-    async def clean_ext_db_files(self, ext_id: str) -> bool:
+    async def clean_ext_db_files(cls, ext_id: str) -> bool:
         """
-        If the extension DB is stored directly on the filesystem (like SQLite) then delete the files and return True.
-        Otherwise do nothing and return False.
+        If the extension DB is stored directly on the filesystem (like SQLite) then
+        delete the files and return True. Otherwise do nothing and return False.
         """
 
         if DB_TYPE == SQLITE:
@@ -375,7 +369,6 @@ class Page(BaseModel, Generic[T]):
 
 class Filter(BaseModel, Generic[TFilterModel]):
     field: str
-    nested: Optional[List[str]]
     op: Operator = Operator.EQ
     values: list[Any]
 
@@ -390,55 +383,36 @@ class Filter(BaseModel, Generic[TFilterModel]):
             split = key[:-1].split("[")
             if len(split) != 2:
                 raise ValueError("Invalid key")
-            field_names = split[0].split(".")
+            field = split[0]
             op = Operator(split[1])
         else:
-            field_names = key.split(".")
+            field = key
             op = Operator("eq")
 
-        field = field_names[0]
-        nested = field_names[1:]
-
         if field in model.__fields__:
             compare_field = model.__fields__[field]
             values = []
             for raw_value in raw_values:
-                # If there is a nested field, pydantic expects a dict, so the raw value is turned into a dict before
-                # and the converted value is extracted afterwards
-                for name in reversed(nested):
-                    raw_value = {name: raw_value}
-
                 validated, errors = compare_field.validate(raw_value, {}, loc="none")
                 if errors:
                     raise ValidationError(errors=[errors], model=model)
-
-                for name in nested:
-                    if isinstance(validated, dict):
-                        validated = validated[name]
-                    else:
-                        validated = getattr(validated, name)
-
                 values.append(validated)
         else:
             raise ValueError("Unknown filter field")
 
-        return cls(field=field, op=op, nested=nested, values=values, model=model)
+        return cls(field=field, op=op, values=values, model=model)
 
     @property
     def statement(self):
-        accessor = self.field
-        if self.nested:
-            for name in self.nested:
-                accessor = f"({accessor} ->> '{name}')"
         if self.model and self.model.__fields__[self.field].type_ == datetime.datetime:
-            placeholder = Compat.timestamp_placeholder
+            placeholder = compat_timestamp_placeholder()
         else:
             placeholder = "?"
         if self.op in (Operator.INCLUDE, Operator.EXCLUDE):
             placeholders = ", ".join([placeholder] * len(self.values))
-            stmt = [f"{accessor} {self.op.as_sql} ({placeholders})"]
+            stmt = [f"{self.field} {self.op.as_sql} ({placeholders})"]
         else:
-            stmt = [f"{accessor} {self.op.as_sql} {placeholder}"] * len(self.values)
+            stmt = [f"{self.field} {self.op.as_sql} {placeholder}"] * len(self.values)
         return " OR ".join(stmt)
 
 
@@ -447,8 +421,8 @@ class Filters(BaseModel, Generic[TFilterModel]):
     Generic helper class for filtering and sorting data.
     For usage in an api endpoint, use the `parse_filters` dependency.
 
-    When constructing this class manually always make sure to pass a model so that the values can be validated.
-    Otherwise, make sure to validate the inputs manually.
+    When constructing this class manually always make sure to pass a model so that
+    the values can be validated. Otherwise, make sure to validate the inputs manually.
     """
 
     filters: List[Filter[TFilterModel]] = []
@@ -491,7 +465,7 @@ class Filters(BaseModel, Generic[TFilterModel]):
         if self.search and self.model:
             if DB_TYPE == POSTGRES:
                 where_stmts.append(
-                    f"lower(concat({f', '.join(self.model.__search_fields__)})) LIKE ?"
+                    f"lower(concat({', '.join(self.model.__search_fields__)})) LIKE ?"
                 )
             elif DB_TYPE == SQLITE:
                 where_stmts.append(

lnbits/decorators.py

@@ -1,7 +1,7 @@
 from http import HTTPStatus
 from typing import Literal, Optional, Type
 
-from fastapi import Query, Request, Security, status
+from fastapi import Query, Request, Security
 from fastapi.exceptions import HTTPException
 from fastapi.openapi.models import APIKey, APIKeyIn
 from fastapi.security import APIKeyHeader, APIKeyQuery
@@ -9,7 +9,7 @@ from fastapi.security.base import SecurityBase
 from pydantic.types import UUID4
 
 from lnbits.core.crud import get_user, get_wallet_for_key
-from lnbits.core.models import User, Wallet
+from lnbits.core.models import User, WalletType, WalletTypeInfo
 from lnbits.db import Filter, Filters, TFilterModel
 from lnbits.requestvars import g
 from lnbits.settings import settings
@@ -25,7 +25,7 @@ class KeyChecker(SecurityBase):
     ):
         self.scheme_name = scheme_name or self.__class__.__name__
         self.auto_error = auto_error
-        self._key_type = "invoice"
+        self._key_type = WalletType.invoice
         self._api_key = api_key
         if api_key:
             key = APIKey(
@@ -39,7 +39,7 @@ class KeyChecker(SecurityBase):
                 name="X-API-KEY",
                 description="Wallet API Key - HEADER",
             )
-        self.wallet = None  # type: ignore
+        self.wallet = None
         self.model: APIKey = key
 
     async def __call__(self, request: Request):
@@ -49,16 +49,16 @@ class KeyChecker(SecurityBase):
                 if self._api_key
                 else request.headers.get("X-API-KEY") or request.query_params["api-key"]
             )
-            # FIXME: Find another way to validate the key. A fetch from DB should be avoided here.
-            #        Also, we should not return the wallet here - thats silly.
-            #        Possibly store it in a Redis DB
-            self.wallet = await get_wallet_for_key(key_value, self._key_type)  # type: ignore
-            if not self.wallet:
+            # FIXME: Find another way to validate the key. A fetch from DB should be
+            #        avoided here. Also, we should not return the wallet here - thats
+            #        silly. Possibly store it in a Redis DB
+            wallet = await get_wallet_for_key(key_value, self._key_type)
+            self.wallet = wallet  # type: ignore
+            if not wallet:
                 raise HTTPException(
                     status_code=HTTPStatus.UNAUTHORIZED,
                     detail="Invalid key or expired key.",
                 )
-
         except KeyError:
             raise HTTPException(
                 status_code=HTTPStatus.BAD_REQUEST, detail="`X-API-KEY` header missing."
@@ -81,7 +81,7 @@ class WalletInvoiceKeyChecker(KeyChecker):
         api_key: Optional[str] = None,
     ):
         super().__init__(scheme_name, auto_error, api_key)
-        self._key_type = "invoice"
+        self._key_type = WalletType.invoice
 
 
 class WalletAdminKeyChecker(KeyChecker):
@@ -100,16 +100,7 @@ class WalletAdminKeyChecker(KeyChecker):
         api_key: Optional[str] = None,
     ):
         super().__init__(scheme_name, auto_error, api_key)
-        self._key_type = "admin"
-
-
-class WalletTypeInfo:
-    wallet_type: int
-    wallet: Wallet
-
-    def __init__(self, wallet_type: int, wallet: Wallet) -> None:
-        self.wallet_type = wallet_type
-        self.wallet = wallet
+        self._key_type = WalletType.admin
 
 
 api_key_header = APIKeyHeader(
@@ -129,11 +120,6 @@ async def get_key_type(
     api_key_header: str = Security(api_key_header),
     api_key_query: str = Security(api_key_query),
 ) -> WalletTypeInfo:
-    # 0: admin
-    # 1: invoice
-    # 2: invalid
-    pathname = r["path"].split("/")[1]
-
     token = api_key_header or api_key_query
 
     if not token:
@@ -142,38 +128,40 @@ async def get_key_type(
             detail="Invoice (or Admin) key required.",
         )
 
-    for typenr, WalletChecker in zip(
-        [0, 1], [WalletAdminKeyChecker, WalletInvoiceKeyChecker]
+    for wallet_type, WalletChecker in zip(
+        [WalletType.admin, WalletType.invoice],
+        [WalletAdminKeyChecker, WalletInvoiceKeyChecker],
     ):
         try:
             checker = WalletChecker(api_key=token)
             await checker.__call__(r)
-            wallet = WalletTypeInfo(typenr, checker.wallet)  # type: ignore
-            if wallet is None or wallet.wallet is None:
+            if checker.wallet is None:
                 raise HTTPException(
                     status_code=HTTPStatus.NOT_FOUND, detail="Wallet does not exist."
                 )
+            wallet = WalletTypeInfo(wallet_type, checker.wallet)
             if (
                 wallet.wallet.user != settings.super_user
                 and wallet.wallet.user not in settings.lnbits_admin_users
             ) and (
                 settings.lnbits_admin_extensions
-                and pathname in settings.lnbits_admin_extensions
+                and r["path"].split("/")[1] in settings.lnbits_admin_extensions
             ):
                 raise HTTPException(
                     status_code=HTTPStatus.FORBIDDEN,
                     detail="User not authorized for this extension.",
                 )
             return wallet
-        except HTTPException as e:
-            if e.status_code == HTTPStatus.BAD_REQUEST:
+        except HTTPException as exc:
+            if exc.status_code == HTTPStatus.BAD_REQUEST:
                 raise
-            elif e.status_code == HTTPStatus.UNAUTHORIZED:
-                # we pass this in case it is not an invoice key, nor an admin key, and then return NOT_FOUND at the end of this block
+            elif exc.status_code == HTTPStatus.UNAUTHORIZED:
+                # we pass this in case it is not an invoice key, nor an admin key,
+                # and then return NOT_FOUND at the end of this block
                 pass
             else:
                 raise
-        except:
+        except Exception:
             raise
     raise HTTPException(
         status_code=HTTPStatus.NOT_FOUND, detail="Wallet does not exist."
@@ -199,7 +187,7 @@ async def require_admin_key(
         # If wallet type is not admin then return the unauthorized status
         # This also covers when the user passes an invalid key type
         raise HTTPException(
-            status_code=status.HTTP_401_UNAUTHORIZED, detail="Admin key required."
+            status_code=HTTPStatus.UNAUTHORIZED, detail="Admin key required."
         )
     else:
         return wallet
@@ -220,11 +208,12 @@ async def require_invoice_key(
 
     wallet = await get_key_type(r, token)
 
-    if wallet.wallet_type > 1:
-        # If wallet type is not invoice then return the unauthorized status
-        # This also covers when the user passes an invalid key type
+    if (
+        wallet.wallet_type != WalletType.admin
+        and wallet.wallet_type != WalletType.invoice
+    ):
         raise HTTPException(
-            status_code=status.HTTP_401_UNAUTHORIZED,
+            status_code=HTTPStatus.UNAUTHORIZED,
             detail="Invoice (or Admin) key required.",
         )
     else:

lnbits/extension_manager.py

@@ -204,7 +204,7 @@ class Extension(NamedTuple):
 
 
 class ExtensionManager:
-    def __init__(self):
+    def __init__(self) -> None:
         p = Path(settings.lnbits_path, "extensions")
         Path(p).mkdir(parents=True, exist_ok=True)
         self._extension_folders: List[Path] = [f for f in p.iterdir() if f.is_dir()]
@@ -426,7 +426,10 @@ class InstallableExtension(BaseModel):
         logger.success(f"Extension {self.name} ({self.installed_version}) installed.")
 
     def nofiy_upgrade(self) -> None:
-        """Update the list of upgraded extensions. The middleware will perform redirects based on this"""
+        """
+        Update the list of upgraded extensions. The middleware will perform
+        redirects based on this
+        """
 
         clean_upgraded_exts = list(
             filter(

lnbits/helpers.py

@@ -4,11 +4,7 @@ from typing import Any, List, Optional, Type
 
 import jinja2
 import shortuuid
-from pydantic.schema import (
-    field_schema,
-    get_flat_models_from_fields,
-    get_model_name_map,
-)
+from pydantic.schema import field_schema
 
 from lnbits.jinja2_templating import Jinja2Templates
 from lnbits.requestvars import g
@@ -90,32 +86,25 @@ def get_current_extension_name() -> str:
         with open(config_path) as json_file:
             config = json.load(json_file)
         ext_name = config["name"]
-    except:
+    except Exception:
         ext_name = extension_director_name
     return ext_name
 
 
 def generate_filter_params_openapi(model: Type[FilterModel], keep_optional=False):
     """
-    Generate openapi documentation for Filters. This is intended to be used along parse_filters (see example)
+    Generate openapi documentation for Filters. This is intended to be used along
+    parse_filters (see example)
     :param model: Filter model
-    :param keep_optional: If false, all parameters will be optional, otherwise inferred from model
+    :param keep_optional: If false, all parameters will be optional,
+    otherwise inferred from model
     """
     fields = list(model.__fields__.values())
-    models = get_flat_models_from_fields(fields, set())
-    namemap = get_model_name_map(models)
     params = []
     for field in fields:
-        schema, definitions, _ = field_schema(field, model_name_map=namemap)
-
-        # Support nested definition
-        if "$ref" in schema:
-            name = schema["$ref"].split("/")[-1]
-            schema = definitions[name]
+        schema, _, _ = field_schema(field, model_name_map={})
 
         description = "Supports Filtering"
-        if schema["type"] == "object":
-            description += f". Nested attributes can be filtered too, e.g. `{field.alias}.[additional].[attributes]`"
         if (
             hasattr(model, "__search_fields__")
             and field.name in model.__search_fields__

lnbits/middleware.py

@@ -7,6 +7,7 @@ from fastapi.responses import HTMLResponse, JSONResponse
 from slowapi import _rate_limit_exceeded_handler
 from slowapi.errors import RateLimitExceeded
 from slowapi.middleware import SlowAPIMiddleware
+from starlette.middleware.gzip import GZipMiddleware
 from starlette.types import ASGIApp, Receive, Scope, Send
 
 from lnbits.core import core_app_extra
@@ -17,7 +18,8 @@ from lnbits.settings import settings
 class InstalledExtensionMiddleware:
     # This middleware class intercepts calls made to the extensions API and:
     #  - it blocks the calls if the extension has been disabled or uninstalled.
-    #  - it redirects the calls to the latest version of the extension if the extension has been upgraded.
+    #  - it redirects the calls to the latest version of the extension
+    #    if the extension has been upgraded.
     #  - otherwise it has no effect
     def __init__(self, app: ASGIApp) -> None:
         self.app = app
@@ -88,9 +90,10 @@ class InstalledExtensionMiddleware:
         self, headers: List[Any], msg: str, status_code: HTTPStatus
     ) -> Union[HTMLResponse, JSONResponse]:
         """
-        Build an HTTP response containing the `msg` as HTTP body and the `status_code` as HTTP code.
-        If the `accept` HTTP header is present int the request and contains the value of `text/html`
-        then return an `HTMLResponse`, otherwise return an `JSONResponse`.
+        Build an HTTP response containing the `msg` as HTTP body and the `status_code`
+        as HTTP code. If the `accept` HTTP header is present int the request and
+        contains the value of `text/html` then return an `HTMLResponse`,
+        otherwise return an `JSONResponse`.
         """
         accept_header: str = next(
             (
@@ -115,9 +118,21 @@ class InstalledExtensionMiddleware:
         )
 
 
+class CustomGZipMiddleware(GZipMiddleware):
+    def __init__(self, *args, exclude_paths=None, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.exclude_paths = exclude_paths or []
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if "path" in scope and scope["path"] in self.exclude_paths:
+            await self.app(scope, receive, send)
+            return
+        await super().__call__(scope, receive, send)
+
+
 class ExtensionsRedirectMiddleware:
-    # Extensions are allowed to specify redirect paths.
-    # A call to a path outside the scope of the extension can be redirected to one of the extension's endpoints.
+    # Extensions are allowed to specify redirect paths. A call to a path outside the
+    # scope of the extension can be redirected to one of the extension's endpoints.
     # Eg: redirect `GET /.well-known` to `GET /lnurlp/api/v1/well-known`
 
     def __init__(self, app: ASGIApp) -> None:
@@ -218,7 +233,8 @@ def add_ip_block_middleware(app: FastAPI):
                 status_code=403,  # Forbidden
                 content={"detail": "IP is blocked"},
             )
-        # this try: except: block is not needed on latest FastAPI (await call_next(request) is enough)
+        # this try: except: block is not needed on latest FastAPI
+        # (await call_next(request) is enough)
         # https://stackoverflow.com/questions/71222144/runtimeerror-no-response-returned-in-fastapi-when-refresh-request
         # TODO: remove after https://github.com/lnbits/lnbits/pull/1609 is merged
         try:

lnbits/server.py

@@ -1,13 +1,10 @@
-import uvloop
-
-uvloop.install()
-
 import multiprocessing as mp
 import time
 from pathlib import Path
 
 import click
 import uvicorn
+from uvicorn.supervisors import ChangeReload
 
 from lnbits.settings import set_cli_settings, settings
 
@@ -41,9 +38,13 @@ def main(
     # create data dir if it does not exist
     Path(settings.lnbits_data_folder).mkdir(parents=True, exist_ok=True)
 
+    # create extension dir if it does not exist
+    Path(settings.lnbits_path, "extensions").mkdir(parents=True, exist_ok=True)
+
     set_cli_settings(host=host, port=port, forwarded_allow_ips=forwarded_allow_ips)
 
-    # this beautiful beast parses all command line arguments and passes them to the uvicorn server
+    # this beautiful beast parses all command line arguments and
+    # passes them to the uvicorn server
     d = dict()
     for a in ctx.args:
         item = a.split("=")
@@ -70,7 +71,14 @@ def main(
         )
 
         server = uvicorn.Server(config=config)
-        process = mp.Process(target=server.run)
+
+        if config.should_reload:
+            sock = config.bind_socket()
+            run = ChangeReload(config, target=server.run, sockets=[sock]).run
+        else:
+            run = server.run
+
+        process = mp.Process(target=run)
         process.start()
         server_restart.wait()
         server_restart.clear()

lnbits/settings.py

@@ -12,21 +12,21 @@ from loguru import logger
 from pydantic import BaseSettings, Extra, Field, validator
 
 
-def list_parse_fallback(v):
-    try:
-        return json.loads(v)
-    except Exception:
-        replaced = v.replace(" ", "")
-        if replaced:
-            return replaced.split(",")
+def list_parse_fallback(v: str):
+    v = v.replace(" ", "")
+    if len(v) > 0:
+        if v.startswith("[") or v.startswith("{"):
+            return json.loads(v)
         else:
-            return []
+            return v.split(",")
+    else:
+        return []
 
 
 class LNbitsSettings(BaseSettings):
     @classmethod
     def validate(cls, val):
-        if type(val) == str:
+        if isinstance(val, str):
             val = val.split(",") if val else []
         return val
 
@@ -90,6 +90,7 @@ class ThemesSettings(LNbitsSettings):
     )  # sneaky sneaky
     lnbits_ad_space_enabled: bool = Field(default=False)
     lnbits_allowed_currencies: List[str] = Field(default=[])
+    lnbits_default_accounting_currency: Optional[str] = Field(default=None)
 
 
 class OpsSettings(LNbitsSettings):
@@ -113,7 +114,9 @@ class SecuritySettings(LNbitsSettings):
     lnbits_watchdog_interval: int = Field(default=60)
     lnbits_watchdog_delta: int = Field(default=1_000_000)
     lnbits_status_manifest: str = Field(
-        default="https://raw.githubusercontent.com/lnbits/lnbits-status/main/manifest.json"
+        default=(
+            "https://raw.githubusercontent.com/lnbits/lnbits-status/main/manifest.json"
+        )
     )
 
 
@@ -137,6 +140,12 @@ class CoreLightningFundingSource(LNbitsSettings):
     clightning_rpc: Optional[str] = Field(default=None)
 
 
+class CoreLightningRestFundingSource(LNbitsSettings):
+    corelightning_rest_url: Optional[str] = Field(default=None)
+    corelightning_rest_macaroon: Optional[str] = Field(default=None)
+    corelightning_rest_cert: Optional[str] = Field(default=None)
+
+
 class EclairFundingSource(LNbitsSettings):
     eclair_url: Optional[str] = Field(default=None)
     eclair_pass: Optional[str] = Field(default=None)
@@ -199,7 +208,7 @@ class BoltzExtensionSettings(LNbitsSettings):
 
 
 class LightningSettings(LNbitsSettings):
-    lightning_invoice_expiry: int = Field(default=600)
+    lightning_invoice_expiry: int = Field(default=3600)
 
 
 class FundingSourcesSettings(
@@ -207,6 +216,7 @@ class FundingSourcesSettings(
     LNbitsFundingSource,
     ClicheFundingSource,
     CoreLightningFundingSource,
+    CoreLightningRestFundingSource,
     EclairFundingSource,
     LndRestFundingSource,
     LndGrpcFundingSource,
@@ -245,6 +255,13 @@ class EditableSettings(
             **{k: v for k, v in d.items() if k in inspect.signature(cls).parameters}
         )
 
+    # fixes openapi.json validation, remove field env_names
+    class Config:
+        @staticmethod
+        def schema_extra(schema: dict[str, Any]) -> None:
+            for prop in schema.get("properties", {}).values():
+                prop.pop("env_names", None)
+
 
 class EnvSettings(LNbitsSettings):
     debug: bool = Field(default=False)
@@ -252,6 +269,7 @@ class EnvSettings(LNbitsSettings):
     host: str = Field(default="127.0.0.1")
     port: int = Field(default=5000)
     forwarded_allow_ips: str = Field(default="*")
+    lnbits_title: str = Field(default="LNbits API")
     lnbits_path: str = Field(default=".")
     lnbits_commit: str = Field(default="unknown")
     super_user: str = Field(default="")
@@ -275,6 +293,7 @@ class SuperUserSettings(LNbitsSettings):
             "VoidWallet",
             "FakeWallet",
             "CoreLightningWallet",
+            "CoreLightningRestWallet",
             "LndRestWallet",
             "EclairWallet",
             "LndWallet",
@@ -361,7 +380,8 @@ def send_admin_user_to_saas():
                 logger.success("sent super_user to saas application")
             except Exception as e:
                 logger.error(
-                    f"error sending super_user to saas: {settings.lnbits_saas_callback}. Error: {str(e)}"
+                    "error sending super_user to saas:"
+                    f" {settings.lnbits_saas_callback}. Error: {str(e)}"
                 )
 
 
@@ -381,7 +401,7 @@ try:
         .strip()
         .decode("ascii")
     )
-except:
+except Exception:
     settings.lnbits_commit = "docker"
 
 settings.version = importlib.metadata.version("lnbits")

lnbits/static/css/base.css

@@ -325,7 +325,7 @@ body[data-theme='autumn'].body--dark {
 [data-theme=flamingo] .q-drawer--dark,
 body[data-theme=flamingo].body--dark,
 [data-theme=flamingo] .q-menu--dark {
-  background: #803a45 !important;
+  background: #2f032f !important;
 }
 
 /* IF WANTING TO SET A DARKER BG COLOR IN THE FUTURE
@@ -336,38 +336,38 @@ body[data-theme='flamingo'].body--dark {
 */
 [data-theme=flamingo] .q-card--dark,
 [data-theme=flamingo] .q-stepper--dark {
-  background: #ec7599 !important;
+  background: #bc23bc !important;
 }
 
 [data-theme=flamingo] .bg-primary {
-  background: #d11d53 !important;
+  background: #ff00ff !important;
 }
 [data-theme=flamingo] .text-primary {
-  color: #d11d53 !important;
+  color: #ff00ff !important;
 }
 [data-theme=flamingo] .bg-secondary {
-  background: #db3e6d !important;
+  background: #fda3fd !important;
 }
 [data-theme=flamingo] .text-secondary {
-  color: #db3e6d !important;
+  color: #fda3fd !important;
 }
 [data-theme=flamingo] .bg-dark {
-  background: #803a45 !important;
+  background: #2f032f !important;
 }
 [data-theme=flamingo] .text-dark {
-  color: #803a45 !important;
+  color: #2f032f !important;
 }
 [data-theme=flamingo] .bg-info {
-  background: #ec7599 !important;
+  background: #bc23bc !important;
 }
 [data-theme=flamingo] .text-info {
-  color: #ec7599 !important;
+  color: #bc23bc !important;
 }
 [data-theme=flamingo] .bg-marginal-bg {
-  background: #803a45 !important;
+  background: #311231 !important;
 }
 [data-theme=flamingo] .text-marginal-bg {
-  color: #803a45 !important;
+  color: #311231 !important;
 }
 [data-theme=flamingo] .bg-marginal-text {
   background: rgb(255, 255, 255) !important;
@@ -454,14 +454,6 @@ body[data-theme='monochrome'].body--dark {
   background: #0f47af !important;
 }
 
-[data-theme=flamingo] .q-drawer--dark {
-  background: #e75480 !important;
-}
-
-[data-theme=flamingo] .q-header {
-  background: #e75480 !important;
-}
-
 [v-cloak] {
   display: none;
 }

lnbits/static/i18n/en.js

@@ -29,6 +29,9 @@ window.localisation.en = {
     'This whole wallet will be deleted, the funds will be UNRECOVERABLE.',
   rename_wallet: 'Rename wallet',
   update_name: 'Update name',
+  fiat_tracking: 'Fiat tracking',
+  currency: 'Currency',
+  update_currency: 'Update currency',
   press_to_claim: 'Press to claim bitcoin',
   donate: 'Donate',
   view_github: 'View on GitHub',
@@ -42,6 +45,8 @@ window.localisation.en = {
   runs_on: 'Runs on',
   credit_hint: 'Press Enter to credit account',
   credit_label: '%{denomination} to credit',
+  paste: 'Paste',
+  paste_from_clipboard: 'Paste from clipboard',
   paste_request: 'Paste Request',
   create_invoice: 'Create Invoice',
   camera_tooltip: 'Use camera to scan an invoice/QR',

lnbits/static/js/base.js

@@ -178,7 +178,8 @@ window.LNbits = {
         id: data.id,
         name: data.name,
         adminkey: data.adminkey,
-        inkey: data.inkey
+        inkey: data.inkey,
+        currency: data.currency
       }
       newWallet.msat = data.balance_msat
       newWallet.sat = Math.round(data.balance_msat / 1000)
@@ -203,7 +204,9 @@ window.LNbits = {
         extra: data.extra,
         wallet_id: data.wallet_id,
         webhook: data.webhook,
-        webhook_status: data.webhook_status
+        webhook_status: data.webhook_status,
+        fiat_amount: data.fiat_amount,
+        fiat_currency: data.fiat_currency
       }
 
       obj.date = Quasar.utils.date.formatDate(

lnbits/static/js/bolt11-decoder.js

@@ -22,7 +22,7 @@ function decode(paymentRequest) {
 }
 
 function decodeHumanReadablePart(humanReadablePart) {
-  let prefixes = ['lnbc', 'lntb', 'lnbcrt', 'lnsb']
+  let prefixes = ['lnbc', 'lntb', 'lnbcrt', 'lnsb', 'lntbs']
   let prefix
   prefixes.forEach(value => {
     if (humanReadablePart.substring(0, value.length) === value) {

lnbits/static/js/components.js

@@ -140,6 +140,7 @@ Vue.component('lnbits-extension-list', {
           <q-item-label lines="1" class="text-caption" v-text="$t('extensions')"></q-item-label>
         </q-item-section>
       </q-item>
+      <div class="lt-md q-mt-xl q-mb-xl"></div>
     </q-list>
   `,
   computed: {

lnbits/static/scss/base.scss

@@ -48,11 +48,11 @@ $themes: (
     marginal-text: rgb(255, 255, 255)
   ),
   'flamingo': (
-    primary: #d11d53,
-    secondary: #db3e6d,
-    dark: #803a45,
-    info: #ec7599,
-    marginal-bg: #803a45,
+    primary: #ff00ff,
+    secondary: #fda3fd,
+    dark: #2f032f,
+    info: #bc23bc,
+    marginal-bg: #311231,
     marginal-text: rgb(255, 255, 255)
   ),
   'monochrome': (
@@ -122,14 +122,6 @@ $themes: (
   background: #0f47af !important;
 }
 
-[data-theme='flamingo'] .q-drawer--dark {
-  background: #e75480 !important;
-}
-
-[data-theme='flamingo'] .q-header {
-  background: #e75480 !important;
-}
-
 [v-cloak] {
   display: none;
 }

lnbits/tasks.py

@@ -3,7 +3,7 @@ import time
 import traceback
 import uuid
 from http import HTTPStatus
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 from fastapi.exceptions import HTTPException
 from loguru import logger
@@ -19,6 +19,26 @@ from lnbits.wallets import get_wallet_class
 
 from .core import db
 
+tasks: List[asyncio.Task] = []
+
+
+def create_task(coro):
+    task = asyncio.create_task(coro)
+    tasks.append(task)
+    return task
+
+
+def create_permanent_task(func):
+    return create_task(catch_everything_and_restart(func))
+
+
+def cancel_all_tasks():
+    for task in tasks:
+        try:
+            task.cancel()
+        except Exception as exc:
+            logger.warning(f"error while cancelling task: {str(exc)}")
+
 
 async def catch_everything_and_restart(func):
     try:
@@ -46,8 +66,8 @@ class SseListenersDict(dict):
         self.name = name or f"sse_listener_{str(uuid.uuid4())[:8]}"
 
     def __setitem__(self, key, value):
-        assert type(key) == str, f"{key} is not a string"
-        assert type(value) == asyncio.Queue, f"{value} is not an asyncio.Queue"
+        assert isinstance(key, str), f"{key} is not a string"
+        assert isinstance(value, asyncio.Queue), f"{value} is not an asyncio.Queue"
         logger.trace(f"sse: adding listener {key} to {self.name}. len = {len(self)+1}")
         return super().__setitem__(key, value)
 
@@ -67,8 +87,8 @@ invoice_listeners: Dict[str, asyncio.Queue] = SseListenersDict("invoice_listener
 
 def register_invoice_listener(send_chan: asyncio.Queue, name: Optional[str] = None):
     """
-    A method intended for extensions (and core/tasks.py) to call when they want to be notified about
-    new invoice payments incoming. Will emit all incoming payments.
+    A method intended for extensions (and core/tasks.py) to call when they want to be
+    notified about new invoice payments incoming. Will emit all incoming payments.
     """
     name_unique = f"{name or 'no_name'}_{str(uuid.uuid4())[:8]}"
     logger.trace(f"sse: registering invoice listener {name_unique}")
@@ -127,9 +147,10 @@ async def check_pending_payments():
     while True:
         async with db.connect() as conn:
             logger.info(
-                f"Task: checking all pending payments (incoming={incoming}, outgoing={outgoing}) of last 15 days"
+                f"Task: checking all pending payments (incoming={incoming},"
+                f" outgoing={outgoing}) of last 15 days"
             )
-            start_time: float = time.time()
+            start_time = time.time()
             pending_payments = await get_payments(
                 since=(int(time.time()) - 60 * 60 * 24 * 15),  # 15 days ago
                 complete=False,
@@ -143,15 +164,17 @@ async def check_pending_payments():
                 await payment.check_status(conn=conn)
 
             logger.info(
-                f"Task: pending check finished for {len(pending_payments)} payments (took {time.time() - start_time:0.3f} s)"
+                f"Task: pending check finished for {len(pending_payments)} payments"
+                f" (took {time.time() - start_time:0.3f} s)"
             )
             # we delete expired invoices once upon the first pending check
             if incoming:
                 logger.debug("Task: deleting all expired invoices")
-                start_time: float = time.time()
+                start_time = time.time()
                 await delete_expired_invoices(conn=conn)
                 logger.info(
-                    f"Task: expired invoice deletion finished (took {time.time() - start_time:0.3f} s)"
+                    "Task: expired invoice deletion finished (took"
+                    f" {time.time() - start_time:0.3f} s)"
                 )
 
         # after the first check we will only check outgoing, not incoming

lnbits/utils/cache.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import asyncio
+from time import time
+from typing import Any, NamedTuple, Optional
+
+from loguru import logger
+
+
+class Cached(NamedTuple):
+    value: Any
+    expiry: float
+
+
+class Cache:
+    """
+    Small caching utility providing simple get/set interface (very much like redis)
+    """
+
+    def __init__(self, interval: float = 10) -> None:
+        self.interval = interval
+        self._values: dict[Any, Cached] = {}
+
+    def get(self, key: str, default=None) -> Optional[Any]:
+        cached = self._values.get(key)
+        if cached is not None:
+            if cached.expiry > time():
+                return cached.value
+            else:
+                self._values.pop(key)
+        return default
+
+    def set(self, key: str, value: Any, expiry: float = 10):
+        self._values[key] = Cached(value, time() + expiry)
+
+    def pop(self, key: str, default=None) -> Optional[Any]:
+        cached = self._values.pop(key, None)
+        if cached and cached.expiry > time():
+            return cached.value
+        return default
+
+    async def save_result(self, coro, key: str, expiry: float = 10):
+        """
+        If `key` exists, return its value, otherwise call coro and cache its result
+        """
+        cached = self.get(key)
+        if cached:
+            return cached
+        else:
+            value = await coro()
+            self.set(key, value, expiry=expiry)
+            return value
+
+    async def invalidate_forever(self):
+        while True:
+            try:
+                await asyncio.sleep(self.interval)
+                ts = time()
+                expired = [k for k, v in self._values.items() if v.expiry < ts]
+                for k in expired:
+                    self._values.pop(k)
+            except Exception:
+                logger.error("Error invalidating cache")
+
+
+cache = Cache()

lnbits/utils/exchange_rates.py

@@ -1,9 +1,11 @@
 import asyncio
-from typing import Callable, List, NamedTuple
+from typing import Callable, NamedTuple
 
 import httpx
 from loguru import logger
 
+from lnbits.utils.cache import cache
+
 currencies = {
     "AED": "United Arab Emirates Dirham",
     "AFN": "Afghan Afghani",
@@ -181,6 +183,19 @@ class Provider(NamedTuple):
 
 
 exchange_rate_providers = {
+    # https://binance-docs.github.io/apidocs/spot/en/#symbol-price-ticker
+    "binance": Provider(
+        "Binance",
+        "binance.com",
+        "https://api.binance.com/api/v3/ticker/price?symbol={FROM}{TO}",
+        lambda data, replacements: data["price"],
+    ),
+    "blockchain": Provider(
+        "Blockchain",
+        "blockchain.com",
+        "https://blockchain.info/tobtc?currency={TO}&value=1",
+        lambda data, replacements: 1 / data,
+    ),
     "exir": Provider(
         "Exir",
         "exir.io",
@@ -227,28 +242,6 @@ async def btc_price(currency: str) -> float:
         "TO": currency.upper(),
         "to": currency.lower(),
     }
-    rates: List[float] = []
-    tasks: List[asyncio.Task] = []
-
-    send_channel: asyncio.Queue = asyncio.Queue()
-
-    async def controller():
-        failures = 0
-        while True:
-            rate = await send_channel.get()
-            if rate:
-                rates.append(rate)
-            else:
-                failures += 1
-
-            if len(rates) >= 2 or len(rates) == 1 and failures >= 2:
-                for t in tasks:
-                    t.cancel()
-                break
-            if failures == len(exchange_rate_providers):
-                for t in tasks:
-                    t.cancel()
-                break
 
     async def fetch_price(provider: Provider):
         url = provider.api_url.format(**replacements)
@@ -257,37 +250,33 @@ async def btc_price(currency: str) -> float:
                 r = await client.get(url, timeout=0.5)
                 r.raise_for_status()
                 data = r.json()
-                rate = float(provider.getter(data, replacements))
-                await send_channel.put(rate)
-        except (
-            TypeError,  # CoinMate returns HTTPStatus 200 but no data when a currency pair is not found
-            KeyError,  # Kraken's response dictionary doesn't include keys we look up for
-            httpx.ConnectTimeout,
-            httpx.ConnectError,
-            httpx.ReadTimeout,
-            httpx.HTTPStatusError,  # Some providers throw a 404 when a currency pair is not found
-        ):
-            await send_channel.put(None)
+                return float(provider.getter(data, replacements))
+        except Exception as e:
+            logger.warning(
+                f"Failed to fetch Bitcoin price "
+                f"for {currency} from {provider.name}: {e}"
+            )
+            raise
 
-    asyncio.create_task(controller())
-    for _, provider in exchange_rate_providers.items():
-        tasks.append(asyncio.create_task(fetch_price(provider)))
-
-    try:
-        await asyncio.gather(*tasks)
-    except asyncio.CancelledError:
-        pass
+    results = await asyncio.gather(
+        *[fetch_price(provider) for provider in exchange_rate_providers.values()],
+        return_exceptions=True,
+    )
+    rates = [r for r in results if not isinstance(r, Exception)]
 
     if not rates:
         return 9999999999
     elif len(rates) == 1:
         logger.warning("Could only fetch one Bitcoin price.")
 
-    return sum([rate for rate in rates]) / len(rates)
+    return sum(rates) / len(rates)
 
 
 async def get_fiat_rate_satoshis(currency: str) -> float:
-    return float(100_000_000 / (await btc_price(currency)))
+    price = await cache.save_result(
+        lambda: btc_price(currency), f"btc-price-{currency}"
+    )
+    return float(100_000_000 / price)
 
 
 async def fiat_amount_as_satoshis(amount: float, currency: str) -> int:

lnbits/wallets/__init__.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-# flake8: noqa: F401
 import importlib
 from typing import Optional
 
@@ -8,8 +7,9 @@ from lnbits.settings import settings
 from lnbits.wallets.base import Wallet
 
 from .cliche import ClicheWallet
-from .cln import CoreLightningWallet
-from .cln import CoreLightningWallet as CLightningWallet
+from .corelightning import CoreLightningWallet
+from .corelightning import CoreLightningWallet as CLightningWallet
+from .corelightningrest import CoreLightningRestWallet
 from .eclair import EclairWallet
 from .fake import FakeWallet
 from .lnbits import LNbitsWallet

lnbits/wallets/cliche.py

@@ -36,7 +36,7 @@ class ClicheWallet(Wallet):
             )
         try:
             data = json.loads(r)
-        except:
+        except Exception:
             return StatusResponse(
                 f"Failed to connect to {self.endpoint}, got: '{r[:200]}...'", 0
             )
@@ -55,13 +55,16 @@ class ClicheWallet(Wallet):
             description_hash_str = (
                 description_hash.hex()
                 if description_hash
-                else hashlib.sha256(unhashed_description).hexdigest()
-                if unhashed_description
-                else None
+                else (
+                    hashlib.sha256(unhashed_description).hexdigest()
+                    if unhashed_description
+                    else None
+                )
             )
             ws = create_connection(self.endpoint)
             ws.send(
-                f"create-invoice --msatoshi {amount*1000} --description_hash {description_hash_str}"
+                f"create-invoice --msatoshi {amount*1000} --description_hash"
+                f" {description_hash_str}"
             )
             r = ws.recv()
         else:
@@ -168,11 +171,12 @@ class ClicheWallet(Wallet):
                     try:
                         if data["result"]["status"]:
                             yield data["result"]["payment_hash"]
-                    except:
+                    except Exception:
                         continue
             except Exception as exc:
                 logger.error(
-                    f"lost connection to cliche's invoices stream: '{exc}', retrying in 5 seconds"
+                    f"lost connection to cliche's invoices stream: '{exc}', retrying in"
+                    " 5 seconds"
                 )
                 await asyncio.sleep(5)
                 continue

lnbits/wallets/corelightning.py

@@ -1,14 +1,9 @@
-try:
-    from pyln.client import LightningRpc, RpcError  # type: ignore
-except ImportError:  # pragma: nocover
-    LightningRpc = None
-
 import asyncio
 import random
-from functools import partial, wraps
-from typing import AsyncGenerator, Optional
+from typing import Any, AsyncGenerator, Optional
 
 from loguru import logger
+from pyln.client import LightningRpc, RpcError
 
 from lnbits import bolt11 as lnbits_bolt11
 from lnbits.settings import settings
@@ -23,43 +18,23 @@ from .base import (
 )
 
 
-def async_wrap(func):
-    @wraps(func)
-    async def run(*args, loop=None, executor=None, **kwargs):
-        if loop is None:
-            loop = asyncio.get_event_loop()
-        partial_func = partial(func, *args, **kwargs)
-        return await loop.run_in_executor(executor, partial_func)
-
-    return run
-
-
-def _pay_invoice(ln, payload):
-    return ln.call("pay", payload)
-
-
-def _paid_invoices_stream(ln, last_pay_index):
-    return ln.waitanyinvoice(last_pay_index)
+async def run_sync(func) -> Any:
+    loop = asyncio.get_event_loop()
+    return await loop.run_in_executor(None, func)
 
 
 class CoreLightningWallet(Wallet):
     def __init__(self):
-        if LightningRpc is None:  # pragma: nocover
-            raise ImportError(
-                "The `pyln-client` library must be installed to use `CoreLightningWallet`."
-            )
-
         self.rpc = settings.corelightning_rpc or settings.clightning_rpc
         self.ln = LightningRpc(self.rpc)
 
-        # check if description_hash is supported (from CLN>=v0.11.0)
-        self.supports_description_hash = (
-            "deschashonly" in self.ln.help("invoice")["help"][0]["command"]
-        )
+        # check if description_hash is supported (from corelightning>=v0.11.0)
+        command = self.ln.help("invoice")["help"][0]["command"]  # type: ignore
+        self.supports_description_hash = "deschashonly" in command
 
         # check last payindex so we can listen from that point on
         self.last_pay_index = 0
-        invoices = self.ln.listinvoices()
+        invoices: dict = self.ln.listinvoices()  # type: ignore
         for inv in invoices["invoices"][::-1]:
             if "pay_index" in inv:
                 self.last_pay_index = inv["pay_index"]
@@ -67,7 +42,7 @@ class CoreLightningWallet(Wallet):
 
     async def status(self) -> StatusResponse:
         try:
-            funds = self.ln.listfunds()
+            funds: dict = self.ln.listfunds()  # type: ignore
             return StatusResponse(
                 None, sum([int(ch["our_amount_msat"]) for ch in funds["channels"]])
             )
@@ -88,29 +63,33 @@ class CoreLightningWallet(Wallet):
         try:
             if description_hash and not unhashed_description:
                 raise Unsupported(
-                    "'description_hash' unsupported by CLN, provide 'unhashed_description'"
+                    "'description_hash' unsupported by CoreLightning, provide"
+                    " 'unhashed_description'"
                 )
             if unhashed_description and not self.supports_description_hash:
                 raise Unsupported("unhashed_description")
-            r = self.ln.invoice(
+            r: dict = self.ln.invoice(  # type: ignore
                 msatoshi=msat,
                 label=label,
-                description=unhashed_description.decode()
-                if unhashed_description
-                else memo,
+                description=(
+                    unhashed_description.decode() if unhashed_description else memo
+                ),
                 exposeprivatechannels=True,
-                deschashonly=True
-                if unhashed_description
-                else False,  # we can't pass None here
+                deschashonly=(
+                    True if unhashed_description else False
+                ),  # we can't pass None here
                 expiry=kwargs.get("expiry"),
             )
 
-            if r.get("code") and r.get("code") < 0:
+            if r.get("code") and r.get("code") < 0:  # type: ignore
                 raise Exception(r.get("message"))
 
             return InvoiceResponse(True, r["payment_hash"], r["bolt11"], "")
         except RpcError as exc:
-            error_message = f"CLN method '{exc.method}' failed with '{exc.error.get('message') or exc.error}'."
+            error_message = (
+                f"CoreLightning method '{exc.method}' failed with"
+                f" '{exc.error.get('message') or exc.error}'."  # type: ignore
+            )
             return InvoiceResponse(False, None, None, error_message)
         except Exception as e:
             return InvoiceResponse(False, None, None, str(e))
@@ -123,20 +102,23 @@ class CoreLightningWallet(Wallet):
             return PaymentResponse(False, None, None, None, "invoice already paid")
 
         fee_limit_percent = fee_limit_msat / invoice.amount_msat * 100
-
+        # so fee_limit_percent is applied even on payments with fee < 5000 millisatoshi
+        # (which is default value of exemptfee)
         payload = {
             "bolt11": bolt11,
             "maxfeepercent": f"{fee_limit_percent:.11}",
-            "exemptfee": 0,  # so fee_limit_percent is applied even on payments with fee < 5000 millisatoshi (which is default value of exemptfee)
+            "exemptfee": 0,
         }
         try:
-            wrapped = async_wrap(_pay_invoice)
-            r = await wrapped(self.ln, payload)
+            r = await run_sync(lambda: self.ln.call("pay", payload))
         except RpcError as exc:
             try:
-                error_message = exc.error["attempts"][-1]["fail_reason"]
-            except:
-                error_message = f"CLN method '{exc.method}' failed with '{exc.error.get('message') or exc.error}'."
+                error_message = exc.error["attempts"][-1]["fail_reason"]  # type: ignore
+            except Exception:
+                error_message = (
+                    f"CoreLightning method '{exc.method}' failed with"
+                    f" '{exc.error.get('message') or exc.error}'."  # type: ignore
+                )
             return PaymentResponse(False, None, None, None, error_message)
         except Exception as exc:
             return PaymentResponse(False, None, None, None, str(exc))
@@ -148,8 +130,8 @@ class CoreLightningWallet(Wallet):
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
         try:
-            r = self.ln.listinvoices(payment_hash=checking_id)
-        except:
+            r: dict = self.ln.listinvoices(payment_hash=checking_id)  # type: ignore
+        except RpcError:
             return PaymentStatus(None)
         if not r["invoices"]:
             return PaymentStatus(None)
@@ -161,15 +143,18 @@ class CoreLightningWallet(Wallet):
                 return PaymentStatus(True)
             elif invoice_resp["status"] == "unpaid":
                 return PaymentStatus(None)
-        logger.warning(f"supplied an invalid checking_id: {checking_id}")
+            elif invoice_resp["status"] == "expired":
+                return PaymentStatus(False)
+        else:
+            logger.warning(f"supplied an invalid checking_id: {checking_id}")
         return PaymentStatus(None)
 
     async def get_payment_status(self, checking_id: str) -> PaymentStatus:
         try:
-            r = self.ln.call("listpays", {"payment_hash": checking_id})
-        except:
+            r: dict = self.ln.listpays(payment_hash=checking_id)  # type: ignore
+        except Exception:
             return PaymentStatus(None)
-        if not r["pays"]:
+        if "pays" not in r or not r["pays"]:
             return PaymentStatus(None)
         payment_resp = r["pays"][-1]
 
@@ -183,19 +168,27 @@ class CoreLightningWallet(Wallet):
                 return PaymentStatus(True, fee_msat, payment_resp["preimage"])
             elif status == "failed":
                 return PaymentStatus(False)
-            return PaymentStatus(None)
-        logger.warning(f"supplied an invalid checking_id: {checking_id}")
+            else:
+                return PaymentStatus(None)
+        else:
+            logger.warning(f"supplied an invalid checking_id: {checking_id}")
         return PaymentStatus(None)
 
     async def paid_invoices_stream(self) -> AsyncGenerator[str, None]:
         while True:
             try:
-                wrapped = async_wrap(_paid_invoices_stream)
-                paid = await wrapped(self.ln, self.last_pay_index)
+                paid = await run_sync(
+                    lambda: self.ln.waitanyinvoice(self.last_pay_index, timeout=2)
+                )
                 self.last_pay_index = paid["pay_index"]
                 yield paid["payment_hash"]
+            except RpcError as exc:
+                # only raise if not a timeout
+                if exc.error["code"] != 904:  # type: ignore
+                    raise
             except Exception as exc:
                 logger.error(
-                    f"lost connection to cln invoices stream: '{exc}', retrying in 5 seconds"
+                    f"lost connection to corelightning invoices stream: '{exc}', "
+                    "retrying in 5 seconds"
                 )
                 await asyncio.sleep(5)

lnbits/wallets/corelightningrest.py

@@ -0,0 +1,255 @@
+import asyncio
+import json
+import random
+from typing import AsyncGenerator, Dict, Optional
+
+import httpx
+from loguru import logger
+
+from lnbits import bolt11 as lnbits_bolt11
+from lnbits.settings import settings
+
+from .base import (
+    InvoiceResponse,
+    PaymentResponse,
+    PaymentStatus,
+    StatusResponse,
+    Unsupported,
+    Wallet,
+)
+from .macaroon import load_macaroon
+
+
+class CoreLightningRestWallet(Wallet):
+    def __init__(self):
+        macaroon = settings.corelightning_rest_macaroon
+        assert macaroon, "missing cln-rest macaroon"
+
+        self.macaroon = load_macaroon(macaroon)
+
+        url = settings.corelightning_rest_url
+        if not url:
+            raise Exception("missing url for corelightning-rest")
+        if not macaroon:
+            raise Exception("missing macaroon for corelightning-rest")
+
+        self.url = url[:-1] if url.endswith("/") else url
+        self.url = (
+            f"https://{self.url}" if not self.url.startswith("http") else self.url
+        )
+        self.auth = {
+            "macaroon": self.macaroon,
+            "encodingtype": "hex",
+            "accept": "application/json",
+        }
+
+        self.cert = settings.corelightning_rest_cert or False
+        self.client = httpx.AsyncClient(verify=self.cert, headers=self.auth)
+        self.last_pay_index = 0
+        self.statuses = {
+            "paid": True,
+            "complete": True,
+            "failed": False,
+            "pending": None,
+        }
+
+    async def cleanup(self):
+        try:
+            await self.client.aclose()
+        except RuntimeError as e:
+            logger.warning(f"Error closing wallet connection: {e}")
+
+    async def status(self) -> StatusResponse:
+        r = await self.client.get(f"{self.url}/v1/channel/localremotebal", timeout=5)
+        r.raise_for_status()
+        if r.is_error or "error" in r.json():
+            try:
+                data = r.json()
+                error_message = data["error"]
+            except Exception:
+                error_message = r.text
+            return StatusResponse(
+                f"Failed to connect to {self.url}, got: '{error_message}...'", 0
+            )
+
+        data = r.json()
+        if len(data) == 0:
+            return StatusResponse("no data", 0)
+
+        return StatusResponse(None, int(data.get("localBalance") * 1000))
+
+    async def create_invoice(
+        self,
+        amount: int,
+        memo: Optional[str] = None,
+        description_hash: Optional[bytes] = None,
+        unhashed_description: Optional[bytes] = None,
+        **kwargs,
+    ) -> InvoiceResponse:
+        label = f"lbl{random.random()}"
+        data: Dict = {
+            "amount": amount * 1000,
+            "description": memo,
+            "label": label,
+        }
+        if description_hash and not unhashed_description:
+            raise Unsupported(
+                "'description_hash' unsupported by CoreLightningRest, "
+                "provide 'unhashed_description'"
+            )
+
+        if unhashed_description:
+            data["description"] = unhashed_description.decode("utf-8")
+
+        if kwargs.get("expiry"):
+            data["expiry"] = kwargs["expiry"]
+
+        if kwargs.get("preimage"):
+            data["preimage"] = kwargs["preimage"]
+
+        r = await self.client.post(
+            f"{self.url}/v1/invoice/genInvoice",
+            data=data,
+        )
+
+        if r.is_error or "error" in r.json():
+            try:
+                data = r.json()
+                error_message = data["error"]
+            except Exception:
+                error_message = r.text
+
+            return InvoiceResponse(False, None, None, error_message)
+
+        data = r.json()
+        assert "payment_hash" in data
+        assert "bolt11" in data
+        # NOTE: use payment_hash when corelightning-rest updates and supports it
+        # return InvoiceResponse(True, data["payment_hash"], data["bolt11"], None)
+        return InvoiceResponse(True, label, data["bolt11"], None)
+
+    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
+        invoice = lnbits_bolt11.decode(bolt11)
+        fee_limit_percent = fee_limit_msat / invoice.amount_msat * 100
+        r = await self.client.post(
+            f"{self.url}/v1/pay",
+            data={
+                "invoice": bolt11,
+                "maxfeepercent": f"{fee_limit_percent:.11}",
+                "exemptfee": 0,  # so fee_limit_percent is applied even on payments
+                # with fee < 5000 millisatoshi (which is default value of exemptfee)
+            },
+            timeout=None,
+        )
+
+        if r.is_error or "error" in r.json():
+            try:
+                data = r.json()
+                error_message = data["error"]
+            except Exception:
+                error_message = r.text
+            return PaymentResponse(False, None, None, None, error_message)
+
+        data = r.json()
+
+        if data["status"] != "complete":
+            return PaymentResponse(False, None, None, None, "payment failed")
+
+        checking_id = data["payment_hash"]
+        preimage = data["payment_preimage"]
+        fee_msat = data["msatoshi_sent"] - data["msatoshi"]
+
+        return PaymentResponse(
+            self.statuses.get(data["status"]), checking_id, fee_msat, preimage, None
+        )
+
+    async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
+        # get invoice bolt11 from checking_id
+        # corelightning-rest wants the "label" here....
+        # NOTE: We can get rid of all labels and use payment_hash when
+        # corelightning-rest updates and supports it
+        r = await self.client.get(
+            f"{self.url}/v1/invoice/listInvoices",
+            params={"label": checking_id},
+        )
+        try:
+            r.raise_for_status()
+            data = r.json()
+
+            if r.is_error or "error" in data or data.get("invoices") is None:
+                raise Exception("error in cln response")
+            return PaymentStatus(self.statuses.get(data["invoices"][0]["status"]))
+        except Exception as e:
+            logger.error(f"Error getting invoice status: {e}")
+            return PaymentStatus(None)
+
+    async def get_payment_status(self, checking_id: str) -> PaymentStatus:
+        from lnbits.core import get_standalone_payment
+
+        payment = await get_standalone_payment(checking_id)
+        if not payment:
+            raise ValueError(f"Payment with checking_id {checking_id} not found")
+        r = await self.client.get(
+            f"{self.url}/v1/pay/listPays",
+            params={"invoice": payment.bolt11},
+        )
+        try:
+            r.raise_for_status()
+            data = r.json()
+
+            if r.is_error or "error" in data or not data.get("pays"):
+                raise Exception("error in corelightning-rest response")
+
+            pay = data["pays"][0]
+
+            fee_msat, preimage = None, None
+            if self.statuses[pay["status"]]:
+                # cut off "msat" and convert to int
+                fee_msat = -int(pay["amount_sent_msat"][:-4]) - int(
+                    pay["amount_msat"][:-4]
+                )
+                preimage = pay["preimage"]
+
+            return PaymentStatus(self.statuses.get(pay["status"]), fee_msat, preimage)
+        except Exception as e:
+            logger.error(f"Error getting payment status: {e}")
+            return PaymentStatus(None)
+
+    async def paid_invoices_stream(self) -> AsyncGenerator[str, None]:
+        while True:
+            try:
+                url = f"{self.url}/v1/invoice/waitAnyInvoice/{self.last_pay_index}"
+                async with self.client.stream("GET", url, timeout=None) as r:
+                    async for line in r.aiter_lines():
+                        inv = json.loads(line)
+                        if "error" in inv and "message" in inv["error"]:
+                            logger.error("Error in paid_invoices_stream:", inv)
+                            raise Exception(inv["error"]["message"])
+                        try:
+                            paid = inv["status"] == "paid"
+                            self.last_pay_index = inv["pay_index"]
+                            if not paid:
+                                continue
+                        except Exception:
+                            continue
+                        logger.trace(f"paid invoice: {inv}")
+                        yield inv["label"]
+                        # NOTE: use payment_hash when corelightning-rest updates
+                        # and supports it
+                        # payment_hash = inv["payment_hash"]
+                        # yield payment_hash
+                        # hack to return payment_hash if the above shouldn't work
+                        # r = await self.client.get(
+                        #     f"{self.url}/v1/invoice/listInvoices",
+                        #     params={"label": inv["label"]},
+                        # )
+                        # paid_invoce = r.json()
+                        # logger.trace(f"paid invoice: {paid_invoce}")
+                        # yield paid_invoce["invoices"][0]["payment_hash"]
+
+            except Exception as exc:
+                logger.debug(
+                    f"lost connection to corelightning-rest invoices stream: '{exc}', "
+                    "reconnecting..."
+                )
+                await asyncio.sleep(0.02)

lnbits/wallets/eclair.py

@@ -44,13 +44,16 @@ class EclairWallet(Wallet):
         self.client = httpx.AsyncClient(base_url=self.url, headers=self.auth)
 
     async def cleanup(self):
-        await self.client.aclose()
+        try:
+            await self.client.aclose()
+        except RuntimeError as e:
+            logger.warning(f"Error closing wallet connection: {e}")
 
     async def status(self) -> StatusResponse:
         r = await self.client.post("/globalbalance", timeout=5)
         try:
             data = r.json()
-        except:
+        except Exception:
             return StatusResponse(
                 f"Failed to connect to {self.url}, got: '{r.text[:200]}...'", 0
             )
@@ -90,7 +93,7 @@ class EclairWallet(Wallet):
             try:
                 data = r.json()
                 error_message = data["error"]
-            except:
+            except Exception:
                 error_message = r.text
 
             return InvoiceResponse(False, None, None, error_message)
@@ -109,7 +112,7 @@ class EclairWallet(Wallet):
             try:
                 data = r.json()
                 error_message = data["error"]
-            except:
+            except Exception:
                 error_message = r.text
             return PaymentResponse(False, None, None, None, error_message)
 
@@ -133,7 +136,7 @@ class EclairWallet(Wallet):
             try:
                 data = r.json()
                 error_message = data["error"]
-            except:
+            except Exception:
                 error_message = r.text
             return PaymentResponse(None, checking_id, None, preimage, error_message)
 
@@ -172,7 +175,7 @@ class EclairWallet(Wallet):
                 "pending": None,
             }
             return PaymentStatus(statuses.get(data["status"]["type"]))
-        except:
+        except Exception:
             return PaymentStatus(None)
 
     async def get_payment_status(self, checking_id: str) -> PaymentStatus:
@@ -203,7 +206,7 @@ class EclairWallet(Wallet):
             return PaymentStatus(
                 statuses.get(data["status"]["type"]), fee_msat, preimage
             )
-        except:
+        except Exception:
             return PaymentStatus(None)
 
     async def paid_invoices_stream(self) -> AsyncGenerator[str, None]:
@@ -222,6 +225,7 @@ class EclairWallet(Wallet):
 
             except Exception as exc:
                 logger.error(
-                    f"lost connection to eclair invoices stream: '{exc}', retrying in 5 seconds"
+                    f"lost connection to eclair invoices stream: '{exc}'"
+                    "retrying in 5 seconds"
                 )
                 await asyncio.sleep(5)

lnbits/wallets/fake.py

@@ -31,7 +31,8 @@ class FakeWallet(Wallet):
 
     async def status(self) -> StatusResponse:
         logger.info(
-            "FakeWallet funding source is for using LNbits as a centralised, stand-alone payment system with brrrrrr."
+            "FakeWallet funding source is for using LNbits as a centralised,"
+            " stand-alone payment system with brrrrrr."
         )
         return StatusResponse(None, 1000000000)
 
@@ -61,7 +62,7 @@ class FakeWallet(Wallet):
             data["tags_set"] = ["h"]
             data["description_hash"] = description_hash
         elif unhashed_description:
-            data["tags_set"] = ["d"]
+            data["tags_set"] = ["h"]
             data["description_hash"] = hashlib.sha256(unhashed_description).digest()
         else:
             data["tags_set"] = ["d"]

lnbits/wallets/lnbits.py

@@ -32,7 +32,10 @@ class LNbitsWallet(Wallet):
         self.client = httpx.AsyncClient(base_url=self.endpoint, headers=self.key)
 
     async def cleanup(self):
-        await self.client.aclose()
+        try:
+            await self.client.aclose()
+        except RuntimeError as e:
+            logger.warning(f"Error closing wallet connection: {e}")
 
     async def status(self) -> StatusResponse:
         try:
@@ -44,7 +47,7 @@ class LNbitsWallet(Wallet):
 
         try:
             data = r.json()
-        except:
+        except Exception:
             return StatusResponse(
                 f"Failed to connect to {self.endpoint}, got: '{r.text[:200]}...'", 0
             )
@@ -96,7 +99,7 @@ class LNbitsWallet(Wallet):
 
         if r.is_error:
             error_message = r.json()["detail"]
-            return PaymentResponse(None, None, None, None, error_message)
+            return PaymentResponse(False, None, None, None, error_message)
         else:
             data = r.json()
             checking_id = data["payment_hash"]
@@ -114,14 +117,14 @@ class LNbitsWallet(Wallet):
             if r.is_error:
                 return PaymentStatus(None)
             return PaymentStatus(r.json()["paid"])
-        except:
+        except Exception:
             return PaymentStatus(None)
 
     async def get_payment_status(self, checking_id: str) -> PaymentStatus:
         r = await self.client.get(url=f"/api/v1/payments/{checking_id}")
 
         if r.is_error:
-            return PaymentStatus(None)
+            return PaymentStatus(False)
         data = r.json()
         if "paid" not in data and "details" not in data:
             return PaymentStatus(None)

lnbits/wallets/lndgrpc.py

@@ -1,10 +1,3 @@
-imports_ok = True
-try:
-    import grpc
-    from grpc import RpcError
-except ImportError:  # pragma: nocover
-    imports_ok = False
-
 import asyncio
 import base64
 import hashlib
@@ -13,14 +6,6 @@ from typing import AsyncGenerator, Dict, Optional
 
 from loguru import logger
 
-from .macaroon import AESCipher, load_macaroon
-
-if imports_ok:
-    import lnbits.wallets.lnd_grpc_files.lightning_pb2 as ln
-    import lnbits.wallets.lnd_grpc_files.lightning_pb2_grpc as lnrpc
-    import lnbits.wallets.lnd_grpc_files.router_pb2 as router
-    import lnbits.wallets.lnd_grpc_files.router_pb2_grpc as routerrpc
-
 from lnbits.settings import settings
 
 from .base import (
@@ -30,6 +15,20 @@ from .base import (
     StatusResponse,
     Wallet,
 )
+from .macaroon import AESCipher, load_macaroon
+
+imports_ok = True
+try:
+    import grpc
+    from grpc import RpcError
+except ImportError:  # pragma: nocover
+    imports_ok = False
+
+if imports_ok:
+    import lnbits.wallets.lnd_grpc_files.lightning_pb2 as ln
+    import lnbits.wallets.lnd_grpc_files.lightning_pb2_grpc as lnrpc
+    import lnbits.wallets.lnd_grpc_files.router_pb2 as router
+    import lnbits.wallets.lnd_grpc_files.router_pb2_grpc as routerrpc
 
 
 def get_ssl_context(cert_path: str):
@@ -83,7 +82,7 @@ def hex_to_b64(hex_str: str) -> str:
 def hex_to_bytes(hex_str: str) -> bytes:
     try:
         return bytes.fromhex(hex_str)
-    except:
+    except Exception:
         return b""
 
 
@@ -101,7 +100,8 @@ class LndWallet(Wallet):
     def __init__(self):
         if not imports_ok:  # pragma: nocover
             raise ImportError(
-                "The `grpcio` and `protobuf` library must be installed to use `GRPC LndWallet`. Alternatively try using the LndRESTWallet."
+                "The `grpcio` and `protobuf` library must be installed to use `GRPC"
+                " LndWallet`. Alternatively try using the LndRESTWallet."
             )
 
         endpoint = settings.lnd_grpc_endpoint
@@ -289,7 +289,7 @@ class LndWallet(Wallet):
                         bytes_to_hex(payment.htlcs[-1].preimage),
                     )
                 return PaymentStatus(statuses[payment.status])
-        except:  # most likely the payment wasn't found
+        except Exception:  # most likely the payment wasn't found
             return PaymentStatus(None)
 
         return PaymentStatus(None)
@@ -306,6 +306,7 @@ class LndWallet(Wallet):
                     yield checking_id
             except Exception as exc:
                 logger.error(
-                    f"lost connection to lnd invoices stream: '{exc}', retrying in 5 seconds"
+                    f"lost connection to lnd invoices stream: '{exc}', "
+                    "retrying in 5 seconds"
                 )
                 await asyncio.sleep(5)

lnbits/wallets/lndrest.py

@@ -48,7 +48,8 @@ class LndRestWallet(Wallet):
 
         if not cert:
             logger.warning(
-                "no certificate for lndrest provided, this only works if you have a publicly issued certificate"
+                "no certificate for lndrest provided, this only works if you have a"
+                " publicly issued certificate"
             )
 
         endpoint = endpoint[:-1] if endpoint.endswith("/") else endpoint
@@ -69,7 +70,10 @@ class LndRestWallet(Wallet):
         )
 
     async def cleanup(self):
-        await self.client.aclose()
+        try:
+            await self.client.aclose()
+        except RuntimeError as e:
+            logger.warning(f"Error closing wallet connection: {e}")
 
     async def status(self) -> StatusResponse:
         try:
@@ -198,7 +202,7 @@ class LndRestWallet(Wallet):
                         )
                     else:
                         return PaymentStatus(None)
-                except:
+                except Exception:
                     continue
 
         return PaymentStatus(None)
@@ -213,13 +217,14 @@ class LndRestWallet(Wallet):
                             inv = json.loads(line)["result"]
                             if not inv["settled"]:
                                 continue
-                        except:
+                        except Exception:
                             continue
 
                         payment_hash = base64.b64decode(inv["r_hash"]).hex()
                         yield payment_hash
             except Exception as exc:
                 logger.error(
-                    f"lost connection to lnd invoices stream: '{exc}', retrying in 5 seconds"
+                    f"lost connection to lnd invoices stream: '{exc}', retrying in 5"
+                    " seconds"
                 )
                 await asyncio.sleep(5)

lnbits/wallets/lnpay.py

@@ -1,11 +1,8 @@
 import asyncio
 import hashlib
-import json
-from http import HTTPStatus
 from typing import AsyncGenerator, Dict, Optional
 
 import httpx
-from fastapi import HTTPException
 from loguru import logger
 
 from lnbits.settings import settings
@@ -35,7 +32,10 @@ class LNPayWallet(Wallet):
         self.client = httpx.AsyncClient(base_url=self.endpoint, headers=self.auth)
 
     async def cleanup(self):
-        await self.client.aclose()
+        try:
+            await self.client.aclose()
+        except RuntimeError as e:
+            logger.warning(f"Error closing wallet connection: {e}")
 
     async def status(self) -> StatusResponse:
         url = f"/wallet/{self.wallet_key}"
@@ -50,7 +50,8 @@ class LNPayWallet(Wallet):
         data = r.json()
         if data["statusType"]["name"] != "active":
             return StatusResponse(
-                f"Wallet {data['user_label']} (data['id']) not active, but {data['statusType']['name']}",
+                f"Wallet {data['user_label']} (data['id']) not active, but"
+                f" {data['statusType']['name']}",
                 0,
             )
 
@@ -62,7 +63,7 @@ class LNPayWallet(Wallet):
         memo: Optional[str] = None,
         description_hash: Optional[bytes] = None,
         unhashed_description: Optional[bytes] = None,
-        **kwargs,
+        **_,
     ) -> InvoiceResponse:
         data: Dict = {"num_satoshis": f"{amount}"}
         if description_hash:
@@ -99,7 +100,7 @@ class LNPayWallet(Wallet):
 
         try:
             data = r.json()
-        except:
+        except Exception:
             return PaymentResponse(
                 False, None, 0, None, f"Got invalid JSON: {r.text[:200]}"
             )
@@ -136,25 +137,26 @@ class LNPayWallet(Wallet):
             yield value
 
     async def webhook_listener(self):
+        logger.error("LNPay webhook listener disabled.")
+        return
         # TODO: request.get_data is undefined, was it something with Flask or quart?
         # probably issue introduced when refactoring?
-        text: str = await request.get_data()  # type: ignore
-        try:
-            data = json.loads(text)
-        except json.decoder.JSONDecodeError:
-            logger.error(f"got something wrong on lnpay webhook endpoint: {text[:200]}")
-            data = None
-        if (
-            type(data) is not dict
-            or "event" not in data
-            or data["event"].get("name") != "wallet_receive"
-        ):
-            raise HTTPException(status_code=HTTPStatus.NO_CONTENT)
+        # text: str = await request.get_data()
+        # try:
+        #     data = json.loads(text)
+        # except json.decoder.JSONDecodeError:
+        #     logger.error(f"error on lnpay webhook endpoint: {text[:200]}")
+        #     data = None
+        # if (
+        #     type(data) is not dict
+        #     or "event" not in data
+        #     or data["event"].get("name") != "wallet_receive"
+        # ):
+        #     raise HTTPException(status_code=HTTPStatus.NO_CONTENT)
 
-        lntx_id = data["data"]["wtx"]["lnTx"]["id"]
-        r = await self.client.get(f"/lntx/{lntx_id}?fields=settled")
-        data = r.json()
-        if data["settled"]:
-            await self.queue.put(lntx_id)
-
-        raise HTTPException(status_code=HTTPStatus.NO_CONTENT)
+        # lntx_id = data["data"]["wtx"]["lnTx"]["id"]
+        # r = await self.client.get(f"/lntx/{lntx_id}?fields=settled")
+        # data = r.json()
+        # if data["settled"]:
+        #     await self.queue.put(lntx_id)
+        # raise HTTPException(status_code=HTTPStatus.NO_CONTENT)

lnbits/wallets/lntips.py

@@ -33,13 +33,16 @@ class LnTipsWallet(Wallet):
         self.client = httpx.AsyncClient(base_url=self.endpoint, headers=self.auth)
 
     async def cleanup(self):
-        await self.client.aclose()
+        try:
+            await self.client.aclose()
+        except RuntimeError as e:
+            logger.warning(f"Error closing wallet connection: {e}")
 
     async def status(self) -> StatusResponse:
         r = await self.client.get("/api/v1/balance", timeout=40)
         try:
             data = r.json()
-        except:
+        except Exception:
             return StatusResponse(
                 f"Failed to connect to {self.endpoint}, got: '{r.text[:200]}...'", 0
             )
@@ -73,7 +76,7 @@ class LnTipsWallet(Wallet):
             try:
                 data = r.json()
                 error_message = data["message"]
-            except:
+            except Exception:
                 error_message = r.text
 
             return InvoiceResponse(False, None, None, error_message)
@@ -96,7 +99,7 @@ class LnTipsWallet(Wallet):
             try:
                 data = r.json()
                 error_message = data["error"]
-            except:
+            except Exception:
                 error_message = r.text
             return PaymentResponse(False, None, 0, None, error_message)
 
@@ -117,7 +120,7 @@ class LnTipsWallet(Wallet):
 
             data = r.json()
             return PaymentStatus(data["paid"])
-        except:
+        except Exception:
             return PaymentStatus(None)
 
     async def get_payment_status(self, checking_id: str) -> PaymentStatus:
@@ -132,7 +135,7 @@ class LnTipsWallet(Wallet):
 
             paid_to_status = {False: None, True: True}
             return PaymentStatus(paid_to_status[data.get("paid")])
-        except:
+        except Exception:
             return PaymentStatus(None)
 
     async def paid_invoices_stream(self) -> AsyncGenerator[str, None]:
@@ -151,7 +154,7 @@ class LnTipsWallet(Wallet):
                             inv = json.loads(data)
                             if not inv.get("payment_hash"):
                                 continue
-                        except:
+                        except Exception:
                             continue
                         yield inv["payment_hash"]
             except Exception:
@@ -161,6 +164,7 @@ class LnTipsWallet(Wallet):
             # since the backend is expected to drop the connection after 90s
             if last_connected is None or time.time() - last_connected < 10:
                 logger.error(
-                    f"lost connection to {self.endpoint}/api/v1/invoicestream, retrying in 5 seconds"
+                    f"lost connection to {self.endpoint}/api/v1/invoicestream, retrying"
+                    " in 5 seconds"
                 )
                 await asyncio.sleep(5)

lnbits/wallets/macaroon/__init__.py

@@ -1 +1 @@
-from .macaroon import AESCipher, load_macaroon  # noqa: F401
+from .macaroon import AESCipher, load_macaroon

lnbits/wallets/macaroon/macaroon.py

@@ -18,16 +18,23 @@ def load_macaroon(macaroon: str) -> str:
     :rtype: str
     """
 
-    # if the macaroon is a file path, load it
+    # if the macaroon is a file path, load it and return hex version
     if macaroon.split(".")[-1] == "macaroon":
         with open(macaroon, "rb") as f:
             macaroon_bytes = f.read()
             return macaroon_bytes.hex()
     else:
+        # if macaroon is a provided string
+        # check if it is hex, if so, return
+        try:
+            bytes.fromhex(macaroon)
+            return macaroon
+        except ValueError:
+            pass
         # convert the bas64 macaroon to hex
         try:
             macaroon = base64.b64decode(macaroon).hex()
-        except:
+        except Exception:
             pass
     return macaroon
 
@@ -52,7 +59,7 @@ class AESCipher:
         return data + (chr(length) * length).encode()
 
     def unpad(self, data):
-        return data[: -(data[-1] if type(data[-1]) == int else ord(data[-1]))]
+        return data[: -(data[-1] if isinstance(data[-1], int) else ord(data[-1]))]
 
     @property
     def passphrase(self):

lnbits/wallets/opennode.py

@@ -1,10 +1,7 @@
 import asyncio
-import hmac
-from http import HTTPStatus
 from typing import AsyncGenerator, Optional
 
 import httpx
-from fastapi import HTTPException
 from loguru import logger
 
 from lnbits.settings import settings
@@ -37,7 +34,10 @@ class OpenNodeWallet(Wallet):
         self.client = httpx.AsyncClient(base_url=self.endpoint, headers=self.auth)
 
     async def cleanup(self):
-        await self.client.aclose()
+        try:
+            await self.client.aclose()
+        except RuntimeError as e:
+            logger.warning(f"Error closing wallet connection: {e}")
 
     async def status(self) -> StatusResponse:
         try:
@@ -133,18 +133,20 @@ class OpenNodeWallet(Wallet):
             yield value
 
     async def webhook_listener(self):
+        logger.error("webhook listener for opennode is disabled.")
+        return
         # TODO: request.form is undefined, was it something with Flask or quart?
         # probably issue introduced when refactoring?
-        data = await request.form  # type: ignore
-        if "status" not in data or data["status"] != "paid":
-            raise HTTPException(status_code=HTTPStatus.NO_CONTENT)
+        # data = await request.form  # type: ignore
+        # if "status" not in data or data["status"] != "paid":
+        #     raise HTTPException(status_code=HTTPStatus.NO_CONTENT)
 
-        charge_id = data["id"]
-        x = hmac.new(self.auth["Authorization"].encode("ascii"), digestmod="sha256")
-        x.update(charge_id.encode("ascii"))
-        if x.hexdigest() != data["hashed_order"]:
-            logger.error("invalid webhook, not from opennode")
-            raise HTTPException(status_code=HTTPStatus.NO_CONTENT)
+        # charge_id = data["id"]
+        # x = hmac.new(self.auth["Authorization"].encode("ascii"), digestmod="sha256")
+        # x.update(charge_id.encode("ascii"))
+        # if x.hexdigest() != data["hashed_order"]:
+        #     logger.error("invalid webhook, not from opennode")
+        #     raise HTTPException(status_code=HTTPStatus.NO_CONTENT)
 
-        await self.queue.put(charge_id)
-        raise HTTPException(status_code=HTTPStatus.NO_CONTENT)
+        # await self.queue.put(charge_id)
+        # raise HTTPException(status_code=HTTPStatus.NO_CONTENT)

lnbits/wallets/spark.py

@@ -37,13 +37,17 @@ class SparkWallet(Wallet):
         )
 
     async def cleanup(self):
-        await self.client.aclose()
+        try:
+            await self.client.aclose()
+        except RuntimeError as e:
+            logger.warning(f"Error closing wallet connection: {e}")
 
     def __getattr__(self, key):
         async def call(*args, **kwargs):
             if args and kwargs:
                 raise TypeError(
-                    f"must supply either named arguments or a list of arguments, not both: {args} {kwargs}"
+                    "must supply either named arguments or a list of arguments, not"
+                    f" both: {args} {kwargs}"
                 )
             elif args:
                 params = args
@@ -70,7 +74,7 @@ class SparkWallet(Wallet):
 
             try:
                 data = r.json()
-            except:
+            except Exception:
                 raise UnknownError(r.text)
 
             if r.is_error:
@@ -158,7 +162,8 @@ class SparkWallet(Wallet):
 
             if len(pays) > 1:
                 raise SparkError(
-                    f"listpays({payment_hash}) returned an unexpected response: {listpays}"
+                    f"listpays({payment_hash}) returned an unexpected response:"
+                    f" {listpays}"
                 )
 
             if pay["status"] == "failed":

lnbits/wallets/void.py

@@ -1,4 +1,4 @@
-from typing import AsyncGenerator, Optional
+from typing import AsyncGenerator
 
 from loguru import logger
 
@@ -7,37 +7,33 @@ from .base import (
     PaymentResponse,
     PaymentStatus,
     StatusResponse,
-    Unsupported,
     Wallet,
 )
 
 
 class VoidWallet(Wallet):
-    async def create_invoice(
-        self,
-        amount: int,
-        memo: Optional[str] = None,
-        description_hash: Optional[bytes] = None,
-        **kwargs,
-    ) -> InvoiceResponse:
-        raise Unsupported("")
+    async def create_invoice(self, *_, **__) -> InvoiceResponse:
+        return InvoiceResponse(
+            ok=False, error_message="VoidWallet cannot create invoices."
+        )
 
     async def status(self) -> StatusResponse:
         logger.warning(
-            (
-                "This backend does nothing, it is here just as a placeholder, you must configure an "
-                "actual backend before being able to do anything useful with LNbits."
-            )
+            "This backend does nothing, it is here just as a placeholder, you must"
+            " configure an actual backend before being able to do anything useful with"
+            " LNbits."
         )
         return StatusResponse(None, 0)
 
-    async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
-        raise Unsupported("")
+    async def pay_invoice(self, *_, **__) -> PaymentResponse:
+        return PaymentResponse(
+            ok=False, error_message="VoidWallet cannot pay invoices."
+        )
 
-    async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
+    async def get_invoice_status(self, *_, **__) -> PaymentStatus:
         return PaymentStatus(None)
 
-    async def get_payment_status(self, checking_id: str) -> PaymentStatus:
+    async def get_payment_status(self, *_, **__) -> PaymentStatus:
         return PaymentStatus(None)
 
     async def paid_invoices_stream(self) -> AsyncGenerator[str, None]:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "lnbits"
-version = "0.10.9"
+version = "0.10.10"
 description = "LNbits, free and open-source Lightning wallet and accounts system."
 authors = ["Alan Bits <alan@lnbits.com>"]
 
@@ -10,7 +10,7 @@ bech32 = "1.2.0"
 bitstring = "3.1.9"
 click = "8.0.4"
 ecdsa = "0.18.0"
-embit = "0.4.9"
+embit = "^0.7.0"
 fastapi = "0.83.0"
 httpx = "0.23.0"
 jinja2 = "3.0.1"
@@ -31,23 +31,21 @@ grpcio = "1.51.1"
 protobuf = "4.21.12"
 Cerberus = "1.3.4"
 async-timeout = "4.0.2"
-pyln-client = "0.11.1"
+pyln-client = "23.8"
 cashu = "0.9.0"
 slowapi = "^0.1.7"
 
 [tool.poetry.group.dev.dependencies]
-flake8 = "^6.0.0"
-isort = "^5.12.0"
 pytest = "^7.1.2"
-mock = "^4.0.3"
-black = "^22.6.0"
+black = "^23.7.0"
 pytest-asyncio = "^0.19.0"
-pytest-cov = "^3.0.0"
-mypy = "^0.971"
+pytest-cov = "^4.1.0"
+mypy = "^1.5.1"
 types-protobuf = "^3.19.22"
-pylint = "^2.17.2"
 pre-commit = "^3.2.2"
 types-mock = "^5.0.0.6"
+openapi-spec-validator = "^0.5.5"
+ruff = "^0.0.284"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
@@ -55,9 +53,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.scripts]
 lnbits = "lnbits.server:main"
-
-[tool.isort]
-profile = "black"
+lnbits-cli = "lnbits.commands:main"
 
 [tool.pyright]
 include = [
@@ -74,6 +70,7 @@ files = "lnbits"
 exclude = [
   "^lnbits/wallets/lnd_grpc_files",
   "^lnbits/extensions",
+  "^lnbits/upgrades",
 ]
 
 [[tool.mypy.overrides]]
@@ -93,44 +90,81 @@ module = [
   "ecdsa.*",
   "psycopg2.*",
   "pyngrok.*",
+  "pyln.client.*",
 ]
 ignore_missing_imports = "True"
 
 [tool.pytest.ini_options]
+log_cli = true
 addopts = "--durations=1 -s --cov=lnbits --cov-report=xml"
 testpaths = [
   "tests"
 ]
 
-[tool.pylint.'MESSAGES CONTROL']
-max-line-length = 300
-disable = "all"
-enable = [
-    "assignment-from-none",
-    "chained-comparison",
-    "consider-merging-isinstance",
-    "consider-using-dict-comprehension",
-    "consider-using-dict-items",
-    "consider-using-f-string",
-    "consider-using-in",
-    "dangerous-default-value",
-    "inconsistent-return-statements",
-    "lost-exception",
-    "pointless-string-statement",
-    "simplifiable-if-statement",
-    "super-init-not-called",
-    "superfluous-parens",
-    "unused-variable",
-    "use-list-literal",
-    "useless-else-on-loop",
-    "useless-object-inheritance",
+[tool.black]
+line-length = 88
+# use upcoming new features
+# preview = true
+extend-exclude = """(
+    lnbits/static
+    | lnbits/extensions
+    | lnbits/upgrades
+    | lnbits/wallets/lnd_grpc_files
+)"""
+
+[tool.ruff]
+# Same as Black. + 10% rule of black
+line-length = 88
+
+# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
+# (`I`) is for `isort`.
+select = ["E", "F", "I"]
+ignore = [
+    "E402", # Module level import not at top of file
 ]
 
-[tool.pylint.MASTER]
-ignore-paths = [
-  "^lnbits/extensions/.*$",
-  "^lnbits/upgrades/.*$",
-  "^lnbits/wallets/lnd_grpc_files/.*$",
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    "lnbits/static",
+    "lnbits/extensions",
+    "lnbits/wallets/lnd_grpc_files",
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
 ]
-fail-under = 10.0
-jobs = 0
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+# Assume Python
+# target-version = "py39"
+
+# Ignore unused imports in __init__.py files.
+[tool.ruff.extend-per-file-ignores]
+"__init__.py" = ["F401", "F403"]
+
+[tool.ruff.mccabe]
+# Unlike Flake8, default to a complexity level of 10.
+max-complexity = 10

tests/conftest.py

@@ -1,15 +1,25 @@
 import asyncio
 
+import uvloop
+
+uvloop.install()  # noqa
+
+import pytest
 import pytest_asyncio
+from fastapi.testclient import TestClient
 from httpx import AsyncClient
 
 from lnbits.app import create_app
-from lnbits.commands import migrate_databases
 from lnbits.core.crud import create_account, create_wallet
-from lnbits.core.views.api import CreateInvoiceData, api_payments_create_invoice
+from lnbits.core.models import CreateInvoice
+from lnbits.core.services import update_wallet_balance
+from lnbits.core.views.api import api_payments_create_invoice
 from lnbits.db import Database
 from lnbits.settings import settings
-from tests.helpers import credit_wallet, get_random_invoice_data, get_real_invoice
+from tests.helpers import get_hold_invoice, get_random_invoice_data, get_real_invoice
+
+# dont install extensions for tests
+settings.lnbits_extensions_default_install = []
 
 
 @pytest_asyncio.fixture(scope="session")
@@ -21,17 +31,11 @@ def event_loop():
 
 # use session scope to run once before and once after all tests
 @pytest_asyncio.fixture(scope="session")
-def app(event_loop):
+async def app():
     app = create_app()
-    # use redefined version of the event loop for scope="session"
-    # loop = asyncio.get_event_loop()
-    loop = event_loop
-    loop.run_until_complete(migrate_databases())
+    await app.router.startup()
     yield app
-    # # get the current event loop and gracefully stop any running tasks
-    # loop = event_loop
-    loop.run_until_complete(loop.shutdown_asyncgens())
-    # loop.close()
+    await app.router.shutdown()
 
 
 @pytest_asyncio.fixture(scope="session")
@@ -41,6 +45,11 @@ async def client(app):
     await client.aclose()
 
 
+@pytest.fixture(scope="session")
+def test_client(app):
+    return TestClient(app)
+
+
 @pytest_asyncio.fixture(scope="session")
 async def db():
     yield Database("database")
@@ -56,13 +65,21 @@ async def from_user():
 async def from_wallet(from_user):
     user = from_user
     wallet = await create_wallet(user_id=user.id, wallet_name="test_wallet_from")
-    await credit_wallet(
+    await update_wallet_balance(
         wallet_id=wallet.id,
         amount=999999999,
     )
     yield wallet
 
 
+@pytest_asyncio.fixture
+async def from_wallet_ws(from_wallet, test_client):
+    # wait a bit in order to avoid receiving topup notification
+    await asyncio.sleep(0.1)
+    with test_client.websocket_connect(f"/api/v1/ws/{from_wallet.id}") as ws:
+        yield ws
+
+
 @pytest_asyncio.fixture(scope="session")
 async def to_user():
     user = await create_account()
@@ -73,13 +90,21 @@ async def to_user():
 async def to_wallet(to_user):
     user = to_user
     wallet = await create_wallet(user_id=user.id, wallet_name="test_wallet_to")
-    await credit_wallet(
+    await update_wallet_balance(
         wallet_id=wallet.id,
         amount=999999999,
     )
     yield wallet
 
 
+@pytest_asyncio.fixture
+async def to_wallet_ws(to_wallet, test_client):
+    # wait a bit in order to avoid receiving topup notification
+    await asyncio.sleep(0.1)
+    with test_client.websocket_connect(f"/api/v1/ws/{to_wallet.id}") as ws:
+        yield ws
+
+
 @pytest_asyncio.fixture(scope="session")
 async def inkey_headers_from(from_wallet):
     wallet = from_wallet
@@ -119,14 +144,21 @@ async def adminkey_headers_to(to_wallet):
 @pytest_asyncio.fixture(scope="session")
 async def invoice(to_wallet):
     data = await get_random_invoice_data()
-    invoiceData = CreateInvoiceData(**data)
+    invoiceData = CreateInvoice(**data)
     invoice = await api_payments_create_invoice(invoiceData, to_wallet)
     yield invoice
     del invoice
 
 
-@pytest_asyncio.fixture(scope="session")
+@pytest_asyncio.fixture(scope="function")
 async def real_invoice():
     invoice = get_real_invoice(100)
     yield {"bolt11": invoice["payment_request"]}
     del invoice
+
+
+@pytest_asyncio.fixture(scope="function")
+async def hold_invoice():
+    invoice = get_hold_invoice(100)
+    yield invoice
+    del invoice

tests/core/test_cache.py

@@ -0,0 +1,62 @@
+import asyncio
+
+import pytest
+
+from lnbits.utils.cache import Cache
+from tests.conftest import pytest_asyncio
+
+key = "foo"
+value = "bar"
+
+
+@pytest_asyncio.fixture
+async def cache():
+    cache = Cache(interval=0.1)
+
+    task = asyncio.create_task(cache.invalidate_forever())
+    yield cache
+    task.cancel()
+
+
+@pytest.mark.asyncio
+async def test_cache_get_set(cache):
+    cache.set(key, value)
+    assert cache.get(key) == value
+    assert cache.get(key, default="default") == value
+    assert cache.get("i-dont-exist", default="default") == "default"
+
+
+@pytest.mark.asyncio
+async def test_cache_expiry(cache):
+    # gets expired by `get` call
+    cache.set(key, value, expiry=0.01)
+    await asyncio.sleep(0.02)
+    assert not cache.get(key)
+
+    # gets expired by invalidation task
+    cache.set(key, value, expiry=0.1)
+    await asyncio.sleep(0.2)
+    assert key not in cache._values
+    assert not cache.get(key)
+
+
+@pytest.mark.asyncio
+async def test_cache_pop(cache):
+    cache.set(key, value)
+    assert cache.pop(key) == value
+    assert not cache.get(key)
+    assert cache.pop(key, default="a") == "a"
+
+
+@pytest.mark.asyncio
+async def test_cache_coro(cache):
+    called = 0
+
+    async def test():
+        nonlocal called
+        called += 1
+        return called
+
+    await cache.save_result(test, key="test")
+    result = await cache.save_result(test, key="test")
+    assert result == called == 1

tests/core/test_db.py

@@ -0,0 +1,12 @@
+from datetime import date
+
+import pytest
+
+from lnbits.db import POSTGRES
+
+
+@pytest.mark.asyncio
+async def test_date_conversion(db):
+    if db.type == POSTGRES:
+        row = await db.fetchone("SELECT now()::date")
+        assert row and isinstance(row[0], date)

tests/core/views/__init__.py

@@ -1 +0,0 @@
-from tests.mocks import WALLET  # noqa: F401

tests/core/views/test_api.py

@@ -5,14 +5,22 @@ from time import time
 import pytest
 
 from lnbits import bolt11
+from lnbits.core.crud import get_standalone_payment, update_payment_details
 from lnbits.core.models import Payment
 from lnbits.core.views.admin_api import api_auditor
 from lnbits.core.views.api import api_payment
 from lnbits.db import DB_TYPE, SQLITE
+from lnbits.settings import settings
 from lnbits.wallets import get_wallet_class
-from tests.conftest import CreateInvoiceData, api_payments_create_invoice
+from tests.conftest import CreateInvoice, api_payments_create_invoice
 
-from ...helpers import get_random_invoice_data, is_fake, pay_real_invoice
+from ...helpers import (
+    cancel_invoice,
+    get_random_invoice_data,
+    is_fake,
+    pay_real_invoice,
+    settle_invoice,
+)
 
 WALLET = get_wallet_class()
 
@@ -77,6 +85,30 @@ async def test_create_invoice(client, inkey_headers_to):
     return invoice
 
 
+@pytest.mark.asyncio
+async def test_create_invoice_fiat_amount(client, inkey_headers_to):
+    data = await get_random_invoice_data()
+    data["unit"] = "EUR"
+    response = await client.post(
+        "/api/v1/payments", json=data, headers=inkey_headers_to
+    )
+    assert response.status_code == 201
+    invoice = response.json()
+    decode = bolt11.decode(invoice["payment_request"])
+    assert decode.amount_msat != data["amount"] * 1000
+    assert decode.payment_hash
+
+    response = await client.get(
+        f"/api/v1/payments/{decode.payment_hash}", headers=inkey_headers_to
+    )
+    assert response.is_success
+    res_data = response.json()
+    extra = res_data["details"]["extra"]
+    assert extra["fiat_amount"] == data["amount"]
+    assert extra["fiat_currency"] == data["unit"]
+    assert extra["fiat_rate"]
+
+
 # check POST /api/v1/payments: invoice creation for internal payments only
 @pytest.mark.asyncio
 async def test_create_internal_invoice(client, inkey_headers_to):
@@ -112,14 +144,26 @@ async def test_create_invoice_custom_expiry(client, inkey_headers_to):
 
 # check POST /api/v1/payments: make payment
 @pytest.mark.asyncio
-async def test_pay_invoice(client, invoice, adminkey_headers_from):
+async def test_pay_invoice(client, from_wallet_ws, invoice, adminkey_headers_from):
     data = {"out": True, "bolt11": invoice["payment_request"]}
     response = await client.post(
         "/api/v1/payments", json=data, headers=adminkey_headers_from
     )
     assert response.status_code < 300
-    assert len(response.json()["payment_hash"]) == 64
-    assert len(response.json()["checking_id"]) > 0
+    invoice = response.json()
+    assert len(invoice["payment_hash"]) == 64
+    assert len(invoice["checking_id"]) > 0
+
+    data = from_wallet_ws.receive_json()
+    assert "wallet_balance" in data
+    payment = Payment(**data["payment"])
+    assert payment.payment_hash == invoice["payment_hash"]
+
+    # websocket from to_wallet cant be tested before https://github.com/lnbits/lnbits/pull/1793
+    # data = to_wallet_ws.receive_json()
+    # assert "wallet_balance" in data
+    # payment = Payment(**data["payment"])
+    # assert payment.payment_hash == invoice["payment_hash"]
 
 
 # check GET /api/v1/payments/<hash>: payment status
@@ -189,18 +233,18 @@ async def test_pay_invoice_adminkey(client, invoice, adminkey_headers_from):
 
 @pytest.mark.asyncio
 async def test_get_payments(client, from_wallet, adminkey_headers_from):
-    # Because sqlite only stores timestamps with milliseconds we have to wait a second to ensure
-    # a different timestamp than previous invoices
-    # due to this limitation both payments (normal and paginated) are tested at the same time as they are almost
-    # identical anyways
+    # Because sqlite only stores timestamps with milliseconds we have to wait a second
+    # to ensure a different timestamp than previous invoices due to this limitation
+    # both payments (normal and paginated) are tested at the same time as they are
+    # almost identical anyways
     if DB_TYPE == SQLITE:
         await asyncio.sleep(1)
     ts = time()
 
     fake_data = [
-        CreateInvoiceData(amount=10, memo="aaaa"),
-        CreateInvoiceData(amount=100, memo="bbbb"),
-        CreateInvoiceData(amount=1000, memo="aabb"),
+        CreateInvoice(amount=10, memo="aaaa"),
+        CreateInvoice(amount=100, memo="bbbb"),
+        CreateInvoice(amount=1000, memo="aabb"),
     ]
 
     for invoice in fake_data:
@@ -264,7 +308,7 @@ async def test_decode_invoice(client, invoice):
 async def test_api_payment_without_key(invoice):
     # check the payment status
     response = await api_payment(invoice["payment_hash"])
-    assert type(response) == dict
+    assert isinstance(response, dict)
     assert response["paid"] is True
     # no key, that's why no "details"
     assert "details" not in response
@@ -277,33 +321,37 @@ async def test_api_payment_with_key(invoice, inkey_headers_from):
     response = await api_payment(
         invoice["payment_hash"], inkey_headers_from["X-Api-Key"]
     )
-    assert type(response) == dict
+    assert isinstance(response, dict)
     assert response["paid"] is True
     assert "details" in response
 
 
 # check POST /api/v1/payments: invoice creation with a description hash
 @pytest.mark.skipif(
-    WALLET.__class__.__name__ in ["CoreLightningWallet"],
+    WALLET.__class__.__name__ in ["CoreLightningWallet", "CoreLightningRestWallet"],
     reason="wallet does not support description_hash",
 )
 @pytest.mark.asyncio
 async def test_create_invoice_with_description_hash(client, inkey_headers_to):
     data = await get_random_invoice_data()
-    descr_hash = hashlib.sha256("asdasdasd".encode()).hexdigest()
+    description = "asdasdasd"
+    descr_hash = hashlib.sha256(description.encode()).hexdigest()
     data["description_hash"] = descr_hash
-
+    data["unhashed_description"] = description.encode().hex()
     response = await client.post(
         "/api/v1/payments", json=data, headers=inkey_headers_to
     )
     invoice = response.json()
-
     invoice_bolt11 = bolt11.decode(invoice["payment_request"])
+
     assert invoice_bolt11.description_hash == descr_hash
-    assert invoice_bolt11.description is None
     return invoice
 
 
+@pytest.mark.skipif(
+    WALLET.__class__.__name__ in ["CoreLightningRestWallet"],
+    reason="wallet does not support unhashed_description",
+)
 @pytest.mark.asyncio
 async def test_create_invoice_with_unhashed_description(client, inkey_headers_to):
     data = await get_random_invoice_data()
@@ -321,6 +369,65 @@ async def test_create_invoice_with_unhashed_description(client, inkey_headers_to
     return invoice
 
 
+@pytest.mark.asyncio
+async def test_update_wallet(client, adminkey_headers_from):
+    name = "new name"
+    currency = "EUR"
+
+    response = await client.patch(
+        "/api/v1/wallet", json={"name": name}, headers=adminkey_headers_from
+    )
+    assert response.status_code == 200
+    assert response.json()["name"] == name
+
+    response = await client.patch(
+        "/api/v1/wallet", json={"currency": currency}, headers=adminkey_headers_from
+    )
+    assert response.status_code == 200
+    assert response.json()["currency"] == currency
+    # name is not changed because updates are partial
+    assert response.json()["name"] == name
+
+
+@pytest.mark.asyncio
+async def test_fiat_tracking(client, adminkey_headers_from):
+    async def create_invoice():
+        data = await get_random_invoice_data()
+        response = await client.post(
+            "/api/v1/payments", json=data, headers=adminkey_headers_from
+        )
+        assert response.is_success
+
+        response = await client.get(
+            f"/api/v1/payments/{response.json()['payment_hash']}",
+            headers=adminkey_headers_from,
+        )
+        assert response.is_success
+        return response.json()["details"]
+
+    async def update_currency(currency):
+        response = await client.patch(
+            "/api/v1/wallet", json={"currency": currency}, headers=adminkey_headers_from
+        )
+        assert response.is_success
+        assert response.json()["currency"] == currency
+
+    await update_currency("")
+
+    settings.lnbits_default_accounting_currency = "USD"
+    payment = await create_invoice()
+    assert payment["extra"]["wallet_fiat_currency"] == "USD"
+    assert payment["extra"]["wallet_fiat_amount"] != payment["amount"]
+    assert payment["extra"]["wallet_fiat_rate"]
+
+    await update_currency("EUR")
+
+    payment = await create_invoice()
+    assert payment["extra"]["wallet_fiat_currency"] == "EUR"
+    assert payment["extra"]["wallet_fiat_amount"] != payment["amount"]
+    assert payment["extra"]["wallet_fiat_rate"]
+
+
 async def get_node_balance_sats():
     audit = await api_auditor()
     return audit["node_balance_msats"] / 1000
@@ -329,7 +436,7 @@ async def get_node_balance_sats():
 @pytest.mark.asyncio
 @pytest.mark.skipif(is_fake, reason="this only works in regtest")
 async def test_pay_real_invoice(
-    client, real_invoice, adminkey_headers_from, inkey_headers_from
+    client, real_invoice, adminkey_headers_from, inkey_headers_from, from_wallet_ws
 ):
     prev_balance = await get_node_balance_sats()
     response = await client.post(
@@ -340,25 +447,265 @@ async def test_pay_real_invoice(
     assert len(invoice["payment_hash"]) == 64
     assert len(invoice["checking_id"]) > 0
 
+    data = from_wallet_ws.receive_json()
+    assert "wallet_balance" in data
+    payment = Payment(**data["payment"])
+    assert payment.payment_hash == invoice["payment_hash"]
+
+    # check the payment status
+    response = await client.get(
+        f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from
+    )
+    assert response.status_code < 300
+    payment_status = response.json()
+    assert payment_status["paid"]
+
+    WALLET = get_wallet_class()
+    status = await WALLET.get_payment_status(invoice["payment_hash"])
+    assert status.paid
+
+    await asyncio.sleep(1)
+    balance = await get_node_balance_sats()
+    assert prev_balance - balance == 100
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_create_real_invoice(client, adminkey_headers_from, inkey_headers_from):
+    prev_balance = await get_node_balance_sats()
+    create_invoice = CreateInvoice(out=False, amount=1000, memo="test")
+    response = await client.post(
+        "/api/v1/payments",
+        json=create_invoice.dict(),
+        headers=adminkey_headers_from,
+    )
+    assert response.status_code < 300
+    invoice = response.json()
+
+    response = await client.get(
+        f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from
+    )
+    assert response.status_code < 300
+    payment_status = response.json()
+    assert not payment_status["paid"]
+
+    async def listen():
+        found_checking_id = False
+        async for checking_id in get_wallet_class().paid_invoices_stream():
+            if checking_id == invoice["checking_id"]:
+                found_checking_id = True
+                return
+        assert found_checking_id
+
+    task = asyncio.create_task(listen())
+    await asyncio.sleep(1)
+    pay_real_invoice(invoice["payment_request"])
+    await asyncio.wait_for(task, timeout=10)
+
+    response = await client.get(
+        f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from
+    )
+    assert response.status_code < 300
+    payment_status = response.json()
+    assert payment_status["paid"]
+
+    await asyncio.sleep(1)
+    balance = await get_node_balance_sats()
+    assert balance - prev_balance == create_invoice.amount
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_pay_real_invoice_set_pending_and_check_state(
+    client, real_invoice, adminkey_headers_from, inkey_headers_from
+):
+    """
+    1. We create an invoice
+    2. We pay it
+    3. We verify that the inoice was paid
+    4. We set the invoice to pending in the database
+    5. We recheck the state of the invoice
+    6. We verify that the invoice is paid
+    """
+    response = await client.post(
+        "/api/v1/payments", json=real_invoice, headers=adminkey_headers_from
+    )
+    assert response.status_code < 300
+    invoice = response.json()
+    assert len(invoice["payment_hash"]) == 64
+    assert len(invoice["checking_id"]) > 0
+
     # check the payment status
     response = await api_payment(
         invoice["payment_hash"], inkey_headers_from["X-Api-Key"]
     )
     assert response["paid"]
 
+    # make sure that the backend also thinks it's paid
+    WALLET = get_wallet_class()
     status = await WALLET.get_payment_status(invoice["payment_hash"])
     assert status.paid
 
-    await asyncio.sleep(0.3)
-    balance = await get_node_balance_sats()
-    assert prev_balance - balance == 100
+    # get the outgoing payment from the db
+    payment = await get_standalone_payment(invoice["payment_hash"])
+    assert payment
+    assert payment.pending is False
+
+    # set the outgoing invoice to pending
+    await update_payment_details(payment.checking_id, pending=True)
+
+    payment_pending = await get_standalone_payment(invoice["payment_hash"])
+    assert payment_pending
+    assert payment_pending.pending is True
+
+    # check the outgoing payment status
+    await payment.check_status()
+
+    payment_not_pending = await get_standalone_payment(invoice["payment_hash"])
+    assert payment_not_pending
+    assert payment_not_pending.pending is False
 
 
 @pytest.mark.asyncio
 @pytest.mark.skipif(is_fake, reason="this only works in regtest")
-async def test_create_real_invoice(client, adminkey_headers_from, inkey_headers_from):
-    prev_balance = await get_node_balance_sats()
-    create_invoice = CreateInvoiceData(out=False, amount=1000, memo="test")
+async def test_pay_hold_invoice_check_pending(
+    client, hold_invoice, adminkey_headers_from
+):
+    preimage, invoice = hold_invoice
+    task = asyncio.create_task(
+        client.post(
+            "/api/v1/payments",
+            json={"bolt11": invoice["payment_request"]},
+            headers=adminkey_headers_from,
+        )
+    )
+    await asyncio.sleep(1)
+
+    # get payment hash from the invoice
+    invoice_obj = bolt11.decode(invoice["payment_request"])
+
+    payment_db = await get_standalone_payment(invoice_obj.payment_hash)
+
+    assert payment_db
+    assert payment_db.pending is True
+
+    settle_invoice(preimage)
+
+    response = await task
+    assert response.status_code < 300
+
+    # check if paid
+
+    await asyncio.sleep(1)
+
+    payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash)
+
+    assert payment_db_after_settlement
+    assert payment_db_after_settlement.pending is False
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_pay_hold_invoice_check_pending_and_fail(
+    client, hold_invoice, adminkey_headers_from
+):
+    preimage, invoice = hold_invoice
+    task = asyncio.create_task(
+        client.post(
+            "/api/v1/payments",
+            json={"bolt11": invoice["payment_request"]},
+            headers=adminkey_headers_from,
+        )
+    )
+    await asyncio.sleep(1)
+
+    # get payment hash from the invoice
+    invoice_obj = bolt11.decode(invoice["payment_request"])
+
+    payment_db = await get_standalone_payment(invoice_obj.payment_hash)
+
+    assert payment_db
+    assert payment_db.pending is True
+
+    preimage_hash = hashlib.sha256(bytes.fromhex(preimage)).hexdigest()
+
+    # cancel the hodl invoice
+    assert preimage_hash == invoice_obj.payment_hash
+    cancel_invoice(preimage_hash)
+
+    response = await task
+    assert response.status_code > 300  # should error
+
+    await asyncio.sleep(1)
+
+    # payment should not be in database anymore
+    payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash)
+    assert payment_db_after_settlement is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_pay_hold_invoice_check_pending_and_fail_cancel_payment_task_in_meantime(
+    client, hold_invoice, adminkey_headers_from
+):
+    preimage, invoice = hold_invoice
+    task = asyncio.create_task(
+        client.post(
+            "/api/v1/payments",
+            json={"bolt11": invoice["payment_request"]},
+            headers=adminkey_headers_from,
+        )
+    )
+    await asyncio.sleep(1)
+
+    # get payment hash from the invoice
+    invoice_obj = bolt11.decode(invoice["payment_request"])
+
+    payment_db = await get_standalone_payment(invoice_obj.payment_hash)
+
+    assert payment_db
+    assert payment_db.pending is True
+
+    # cancel payment task, this simulates the client dropping the connection
+    task.cancel()
+
+    preimage_hash = hashlib.sha256(bytes.fromhex(preimage)).hexdigest()
+
+    assert preimage_hash == invoice_obj.payment_hash
+    cancel_invoice(preimage_hash)
+
+    # check if paid
+    await asyncio.sleep(1)
+
+    # payment should still be in db
+    payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash)
+    assert payment_db_after_settlement is not None
+
+    # status should still be available and be False
+    status = await payment_db.check_status()
+    assert status.paid is False
+
+    # now the payment should be gone after the status check
+    payment_db_after_status_check = await get_standalone_payment(
+        invoice_obj.payment_hash
+    )
+    assert payment_db_after_status_check is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_receive_real_invoice_set_pending_and_check_state(
+    client, adminkey_headers_from, inkey_headers_from
+):
+    """
+    1. We create a real invoice
+    2. We pay it from our wallet
+    3. We check that the inoice was paid with the backend
+    4. We set the invoice to pending in the database
+    5. We recheck the state of the invoice with the backend
+    6. We verify that the invoice is now marked as paid in the database
+    """
+    create_invoice = CreateInvoice(out=False, amount=1000, memo="test")
     response = await client.post(
         "/api/v1/payments",
         json=create_invoice.dict(),
@@ -372,18 +719,51 @@ async def test_create_real_invoice(client, adminkey_headers_from, inkey_headers_
     assert not response["paid"]
 
     async def listen():
-        async for payment_hash in get_wallet_class().paid_invoices_stream():
-            assert payment_hash == invoice["payment_hash"]
-            return
+        found_checking_id = False
+        async for checking_id in get_wallet_class().paid_invoices_stream():
+            if checking_id == invoice["checking_id"]:
+                found_checking_id = True
+                return
+        assert found_checking_id
 
     task = asyncio.create_task(listen())
+    await asyncio.sleep(1)
     pay_real_invoice(invoice["payment_request"])
-    await asyncio.wait_for(task, timeout=3)
+    await asyncio.wait_for(task, timeout=10)
     response = await api_payment(
         invoice["payment_hash"], inkey_headers_from["X-Api-Key"]
     )
     assert response["paid"]
 
-    await asyncio.sleep(0.3)
-    balance = await get_node_balance_sats()
-    assert balance - prev_balance == create_invoice.amount
+    # get the incoming payment from the db
+    payment = await get_standalone_payment(invoice["payment_hash"], incoming=True)
+    assert payment
+    assert payment.pending is False
+
+    # set the incoming invoice to pending
+    await update_payment_details(payment.checking_id, pending=True)
+
+    payment_pending = await get_standalone_payment(
+        invoice["payment_hash"], incoming=True
+    )
+    assert payment_pending
+    assert payment_pending.pending is True
+
+    # check the incoming payment status
+    await payment.check_status()
+
+    payment_not_pending = await get_standalone_payment(
+        invoice["payment_hash"], incoming=True
+    )
+    assert payment_not_pending
+    assert payment_not_pending.pending is False
+
+    # verify we get the same result if we use the checking_id to look up the payment
+    payment_by_checking_id = await get_standalone_payment(
+        payment_not_pending.checking_id, incoming=True
+    )
+
+    assert payment_by_checking_id
+    assert payment_by_checking_id.pending is False
+    assert payment_by_checking_id.bolt11 == payment_not_pending.bolt11
+    assert payment_by_checking_id.payment_hash == payment_not_pending.payment_hash

tests/helpers.py

@@ -1,31 +1,17 @@
 import hashlib
 import json
+import os
 import random
-import secrets
 import string
-from subprocess import PIPE, Popen, run
+import time
+from subprocess import PIPE, Popen, TimeoutExpired
+from typing import Tuple
+
+from loguru import logger
 
-from lnbits.core.crud import create_payment
 from lnbits.wallets import get_wallet_class, set_wallet_class
 
 
-async def credit_wallet(wallet_id: str, amount: int):
-    preimage = secrets.token_hex(32)
-    m = hashlib.sha256()
-    m.update(f"{preimage}".encode())
-    payment_hash = m.hexdigest()
-    await create_payment(
-        wallet_id=wallet_id,
-        payment_request="",
-        payment_hash=payment_hash,
-        checking_id=payment_hash,
-        preimage=preimage,
-        memo=f"funding_test_{get_random_string(5)}",
-        amount=amount,  # msat
-        pending=False,  # not pending, so it will increase the wallet's balance
-    )
-
-
 def get_random_string(N: int = 10):
     return "".join(
         random.SystemRandom().choice(string.ascii_uppercase + string.digits)
@@ -43,46 +29,105 @@ is_fake: bool = WALLET.__class__.__name__ == "FakeWallet"
 is_regtest: bool = not is_fake
 
 
-docker_bitcoin_rpc = "lnbits"
-docker_prefix = "lnbits-legend"
-docker_cmd = "docker exec"
+docker_lightning_cli = [
+    "docker",
+    "exec",
+    "lnbits-legend-lnd-1-1",
+    "lncli",
+    "--network",
+    "regtest",
+    "--rpcserver=lnd-1",
+]
 
-docker_lightning = f"{docker_cmd} {docker_prefix}-lnd-1-1"
-docker_lightning_cli = f"{docker_lightning} lncli --network regtest --rpcserver=lnd-1"
-
-docker_bitcoin = f"{docker_cmd} {docker_prefix}-bitcoind-1-1"
-docker_bitcoin_cli = f"{docker_bitcoin} bitcoin-cli -rpcuser={docker_bitcoin_rpc} -rpcpassword={docker_bitcoin_rpc} -regtest"
+docker_bitcoin_cli = [
+    "docker",
+    "exec",
+    "lnbits-legend-bitcoind-1-1" "bitcoin-cli",
+    "-rpcuser=lnbits",
+    "-rpcpassword=lnbits",
+    "-regtest",
+]
 
 
-def run_cmd(cmd: str) -> str:
-    return run(cmd, shell=True, capture_output=True).stdout.decode("UTF-8").strip()
+def run_cmd(cmd: list) -> str:
+    timeout = 20
+    process = Popen(cmd, stdout=PIPE, stderr=PIPE)
+
+    def process_communication(comm):
+        stdout, stderr = comm
+        output = stdout.decode("utf-8").strip()
+        error = stderr.decode("utf-8").strip()
+        return output, error
+
+    try:
+        now = time.time()
+        output, error = process_communication(process.communicate(timeout=timeout))
+        took = time.time() - now
+        logger.debug(f"ran command output: {output}, error: {error}, took: {took}s")
+        return output
+    except TimeoutExpired:
+        process.kill()
+        output, error = process_communication(process.communicate())
+        logger.error(f"timeout command: {cmd}, output: {output}, error: {error}")
+        raise
 
 
-def run_cmd_json(cmd: str) -> dict:
-    return json.loads(run_cmd(cmd))
+def run_cmd_json(cmd: list) -> dict:
+    output = run_cmd(cmd)
+    try:
+        return json.loads(output) if output else {}
+    except json.decoder.JSONDecodeError:
+        logger.error(f"failed to decode json from cmd `{cmd}`: {output}")
+        raise
+
+
+def get_hold_invoice(sats: int) -> Tuple[str, dict]:
+    preimage = os.urandom(32)
+    preimage_hash = hashlib.sha256(preimage).hexdigest()
+    cmd = docker_lightning_cli.copy()
+    cmd.extend(["addholdinvoice", preimage_hash, str(sats)])
+    json = run_cmd_json(cmd)
+    return preimage.hex(), json
+
+
+def settle_invoice(preimage: str) -> str:
+    cmd = docker_lightning_cli.copy()
+    cmd.extend(["settleinvoice", preimage])
+    return run_cmd(cmd)
+
+
+def cancel_invoice(preimage_hash: str) -> str:
+    cmd = docker_lightning_cli.copy()
+    cmd.extend(["cancelinvoice", preimage_hash])
+    return run_cmd(cmd)
 
 
 def get_real_invoice(sats: int) -> dict:
-    return run_cmd_json(f"{docker_lightning_cli} addinvoice {sats}")
+    cmd = docker_lightning_cli.copy()
+    cmd.extend(["addinvoice", str(sats)])
+    return run_cmd_json(cmd)
 
 
-def pay_real_invoice(invoice: str) -> Popen:
-    return Popen(
-        f"{docker_lightning_cli} payinvoice --force {invoice}",
-        shell=True,
-        stdin=PIPE,
-        stdout=PIPE,
-    )
+def pay_real_invoice(invoice: str) -> str:
+    cmd = docker_lightning_cli.copy()
+    cmd.extend(["payinvoice", "--force", invoice])
+    return run_cmd(cmd)
 
 
 def mine_blocks(blocks: int = 1) -> str:
-    return run_cmd(f"{docker_bitcoin_cli} -generate {blocks}")
+    cmd = docker_bitcoin_cli.copy()
+    cmd.extend(["-generate", str(blocks)])
+    return run_cmd(cmd)
 
 
 def create_onchain_address(address_type: str = "bech32") -> str:
-    return run_cmd(f"{docker_bitcoin_cli} getnewaddress {address_type}")
+    cmd = docker_bitcoin_cli.copy()
+    cmd.extend(["getnewaddress", address_type])
+    return run_cmd(cmd)
 
 
 def pay_onchain(address: str, sats: int) -> str:
     btc = sats * 0.00000001
-    return run_cmd(f"{docker_bitcoin_cli} sendtoaddress {address} {btc}")
+    cmd = docker_bitcoin_cli.copy()
+    cmd.extend(["sendtoaddress", address, str(btc)])
+    return run_cmd(cmd)

tests/mocks.py

@@ -1,77 +0,0 @@
-from mock import AsyncMock
-
-from lnbits import bolt11
-from lnbits.wallets.base import PaymentResponse, PaymentStatus, StatusResponse
-from lnbits.wallets.fake import FakeWallet
-
-from .helpers import WALLET, get_random_string, is_fake
-
-
-# generates an invoice with FakeWallet
-async def generate_mock_invoice(**x):
-    invoice = await FakeWallet.create_invoice(
-        FakeWallet(), amount=10, memo=f"mock invoice {get_random_string()}"
-    )
-    return invoice
-
-
-if is_fake:
-    WALLET.status = AsyncMock(
-        return_value=StatusResponse(
-            "",  # no error
-            1000000,  # msats
-        )
-    )
-
-# Note: if this line is uncommented, invoices will always be generated by FakeWallet
-# WALLET.create_invoice = generate_mock_invoice
-
-# NOTE: This mock fails since it yields the same invoice multiple
-# times which makes the db throw an error due to uniqueness contraints
-# on the checking ID
-
-# # primitive event loop for generate_mock_invoice()
-# def drive(c):
-#     while True:
-#         try:
-#             c.send(None)
-#         except StopIteration as e:
-#             return e.value
-
-# # finally we await it
-# invoice = drive(generate_mock_invoice())
-
-# WALLET.create_invoice = AsyncMock(
-#     return_value=InvoiceResponse(
-#         True,  # ok
-#         invoice.checking_id,  # checking_id (i.e. payment_hash)
-#         invoice.payment_request,  # payment_request
-#         "",  # no error
-#     )
-# )
-
-
-if is_fake:
-
-    def pay_invoice_side_effect(
-        payment_request: str, fee_limit_msat: int
-    ) -> PaymentResponse:
-        invoice = bolt11.decode(payment_request)
-        return PaymentResponse(
-            True,  # ok
-            invoice.payment_hash,  # checking_id (i.e. payment_hash)
-            0,  # fee_msat
-            "",  # no error
-        )
-
-    WALLET.pay_invoice = AsyncMock(side_effect=pay_invoice_side_effect)
-    WALLET.get_invoice_status = AsyncMock(
-        return_value=PaymentStatus(
-            True,  # paid
-        )
-    )
-    WALLET.get_payment_status = AsyncMock(
-        return_value=PaymentStatus(
-            True,  # paid
-        )
-    )

tools/conv.py

@@ -55,7 +55,8 @@ def check_db_versions(sqdb):
             version = dbpost[key]
             if value != version:
                 raise Exception(
-                    f"sqlite database version ({value}) of {key} doesn't match postgres database version {version}"
+                    f"sqlite database version ({value}) of {key} doesn't match postgres"
+                    f" database version {version}"
                 )
 
     connection = postgres.connection
@@ -174,7 +175,10 @@ parser.add_argument(
     dest="sqlite_path",
     const=True,
     nargs="?",
-    help=f"SQLite DB folder *or* single extension db file to migrate. Default: {sqfolder}",
+    help=(
+        "SQLite DB folder *or* single extension db file to migrate. Default:"
+        f" {sqfolder}"
+    ),
     default=sqfolder,
     type=str,
 )