Fix overlapping redirect paths (#2671)

lnbits/core/crud.py

@@ -8,14 +8,14 @@ import shortuuid
 from passlib.context import CryptContext
 
 from lnbits.core.db import db
-from lnbits.core.models import PaymentState
-from lnbits.db import DB_TYPE, SQLITE, Connection, Database, Filters, Page
-from lnbits.extension_manager import (
+from lnbits.core.extensions.models import (
     InstallableExtension,
     PayToEnableInfo,
     UserExtension,
     UserExtensionInfo,
 )
+from lnbits.core.models import PaymentState
+from lnbits.db import DB_TYPE, SQLITE, Connection, Database, Filters, Page
 from lnbits.settings import (
     AdminSettings,
     EditableSettings,
@@ -430,7 +430,7 @@ async def get_installed_extension(
 async def get_installed_extensions(
     active: Optional[bool] = None,
     conn: Optional[Connection] = None,
-) -> List["InstallableExtension"]:
+) -> List[InstallableExtension]:
     rows = await (conn or db).fetchall(
         "SELECT * FROM installed_extensions",
         (),

lnbits/core/extensions/extension_manager.py

@@ -0,0 +1,93 @@
+import asyncio
+import importlib
+
+from loguru import logger
+
+from lnbits.core.crud import (
+    add_installed_extension,
+    delete_installed_extension,
+    get_dbversions,
+    get_installed_extension,
+    update_installed_extension_state,
+)
+from lnbits.core.db import core_app_extra
+from lnbits.core.helpers import migrate_extension_database
+from lnbits.settings import settings
+
+from .models import Extension, InstallableExtension
+
+
+async def install_extension(ext_info: InstallableExtension) -> Extension:
+    extension = Extension.from_installable_ext(ext_info)
+    installed_ext = await get_installed_extension(ext_info.id)
+    ext_info.payments = installed_ext.payments if installed_ext else []
+
+    await ext_info.download_archive()
+
+    ext_info.extract_archive()
+
+    db_version = (await get_dbversions()).get(ext_info.id, 0)
+    await migrate_extension_database(extension, db_version)
+
+    await add_installed_extension(ext_info)
+
+    if extension.is_upgrade_extension:
+        # call stop while the old routes are still active
+        await stop_extension_background_work(ext_info.id)
+
+    return extension
+
+
+async def uninstall_extension(ext_id: str):
+    await stop_extension_background_work(ext_id)
+
+    settings.deactivate_extension_paths(ext_id)
+
+    extension = await get_installed_extension(ext_id)
+    if extension:
+        extension.clean_extension_files()
+    await delete_installed_extension(ext_id=ext_id)
+
+
+async def activate_extension(ext: Extension):
+    core_app_extra.register_new_ext_routes(ext)
+    await update_installed_extension_state(ext_id=ext.code, active=True)
+
+
+async def deactivate_extension(ext_id: str):
+    settings.deactivate_extension_paths(ext_id)
+    await update_installed_extension_state(ext_id=ext_id, active=False)
+
+
+async def stop_extension_background_work(ext_id: str) -> bool:
+    """
+    Stop background work for extension (like asyncio.Tasks, WebSockets, etc).
+    Extensions SHOULD expose a `api_stop()` function.
+    """
+    upgrade_hash = settings.lnbits_upgraded_extensions.get(ext_id, "")
+    ext = Extension(ext_id, True, False, upgrade_hash=upgrade_hash)
+
+    try:
+        logger.info(f"Stopping background work for extension '{ext.module_name}'.")
+        old_module = importlib.import_module(ext.module_name)
+
+        # Extensions must expose an `{ext_id}_stop()` function at the module level
+        # The `api_stop()` function is for backwards compatibility (will be deprecated)
+        stop_fns = [f"{ext_id}_stop", "api_stop"]
+        stop_fn_name = next((fn for fn in stop_fns if hasattr(old_module, fn)), None)
+        assert stop_fn_name, "No stop function found for '{ext.module_name}'"
+
+        stop_fn = getattr(old_module, stop_fn_name)
+        if stop_fn:
+            if asyncio.iscoroutinefunction(stop_fn):
+                await stop_fn()
+            else:
+                stop_fn()
+
+        logger.info(f"Stopped background work for extension '{ext.module_name}'.")
+    except Exception as ex:
+        logger.warning(f"Failed to stop background work for '{ext.module_name}'.")
+        logger.warning(ex)
+        return False
+
+    return True

lnbits/core/extensions/helpers.py

@@ -0,0 +1,56 @@
+import hashlib
+from typing import Any, Optional
+from urllib import request
+
+import httpx
+from loguru import logger
+from packaging import version
+
+from lnbits.settings import settings
+
+
+def version_parse(v: str):
+    """
+    Wrapper for version.parse() that does not throw if the version is invalid.
+    Instead it return the lowest possible version ("0.0.0")
+    """
+    try:
+        return version.parse(v)
+    except Exception:
+        return version.parse("0.0.0")
+
+
+async def github_api_get(url: str, error_msg: Optional[str]) -> Any:
+    headers = {"User-Agent": settings.user_agent}
+    if settings.lnbits_ext_github_token:
+        headers["Authorization"] = f"Bearer {settings.lnbits_ext_github_token}"
+    async with httpx.AsyncClient(headers=headers) as client:
+        resp = await client.get(url)
+        if resp.status_code != 200:
+            logger.warning(f"{error_msg} ({url}): {resp.text}")
+        resp.raise_for_status()
+        return resp.json()
+
+
+def download_url(url, save_path):
+    with request.urlopen(url, timeout=60) as dl_file:
+        with open(save_path, "wb") as out_file:
+            out_file.write(dl_file.read())
+
+
+def file_hash(filename):
+    h = hashlib.sha256()
+    b = bytearray(128 * 1024)
+    mv = memoryview(b)
+    with open(filename, "rb", buffering=0) as f:
+        while n := f.readinto(mv):
+            h.update(mv[:n])
+    return h.hexdigest()
+
+
+def icon_to_github_url(source_repo: str, path: Optional[str]) -> str:
+    if not path:
+        return ""
+    _, _, *rest = path.split("/")
+    tail = "/".join(rest)
+    return f"https://github.com/{source_repo}/raw/main/{tail}"

lnbits/core/extensions/models.py

@@ -0,0 +1,753 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import shutil
+import sys
+import zipfile
+from pathlib import Path
+from typing import Any, NamedTuple, Optional
+
+import httpx
+from loguru import logger
+from pydantic import BaseModel
+
+from lnbits.settings import settings
+
+from .helpers import (
+    download_url,
+    file_hash,
+    github_api_get,
+    icon_to_github_url,
+    version_parse,
+)
+
+
+class ExplicitRelease(BaseModel):
+    id: str
+    name: str
+    version: str
+    archive: str
+    hash: str
+    dependencies: list[str] = []
+    repo: Optional[str]
+    icon: Optional[str]
+    short_description: Optional[str]
+    min_lnbits_version: Optional[str]
+    html_url: Optional[str]  # todo: release_url
+    warning: Optional[str]
+    info_notification: Optional[str]
+    critical_notification: Optional[str]
+    details_link: Optional[str]
+    pay_link: Optional[str]
+
+    def is_version_compatible(self):
+        if not self.min_lnbits_version:
+            return True
+        return version_parse(self.min_lnbits_version) <= version_parse(settings.version)
+
+
+class GitHubRelease(BaseModel):
+    id: str
+    organisation: str
+    repository: str
+
+
+class Manifest(BaseModel):
+    featured: list[str] = []
+    extensions: list[ExplicitRelease] = []
+    repos: list[GitHubRelease] = []
+
+
+class GitHubRepoRelease(BaseModel):
+    name: str
+    tag_name: str
+    zipball_url: str
+    html_url: str
+
+    def details_link(self, source_repo: str) -> str:
+        return f"https://raw.githubusercontent.com/{source_repo}/{self.tag_name}/config.json"
+
+
+class GitHubRepo(BaseModel):
+    stargazers_count: str
+    html_url: str
+    default_branch: str
+
+
+class ExtensionConfig(BaseModel):
+    name: str
+    short_description: str
+    tile: str = ""
+    warning: Optional[str] = ""
+    min_lnbits_version: Optional[str]
+
+    def is_version_compatible(self):
+        if not self.min_lnbits_version:
+            return True
+        return version_parse(self.min_lnbits_version) <= version_parse(settings.version)
+
+    @classmethod
+    async def fetch_github_release_config(
+        cls, org: str, repo: str, tag_name: str
+    ) -> Optional[ExtensionConfig]:
+        config_url = (
+            f"https://raw.githubusercontent.com/{org}/{repo}/{tag_name}/config.json"
+        )
+        error_msg = "Cannot fetch GitHub extension config"
+        config = await github_api_get(config_url, error_msg)
+        return ExtensionConfig.parse_obj(config)
+
+
+class ReleasePaymentInfo(BaseModel):
+    amount: Optional[int] = None
+    pay_link: Optional[str] = None
+    payment_hash: Optional[str] = None
+    payment_request: Optional[str] = None
+
+
+class PayToEnableInfo(BaseModel):
+    required: Optional[bool] = False
+    amount: Optional[int] = None
+    wallet: Optional[str] = None
+
+
+class UserExtensionInfo(BaseModel):
+    paid_to_enable: Optional[bool] = False
+    payment_hash_to_enable: Optional[str] = None
+
+
+class UserExtension(BaseModel):
+    extension: str
+    active: bool
+    extra: Optional[UserExtensionInfo] = None
+
+    @property
+    def is_paid(self) -> bool:
+        if not self.extra:
+            return False
+        return self.extra.paid_to_enable is True
+
+    @classmethod
+    def from_row(cls, data: dict) -> UserExtension:
+        ext = UserExtension(**data)
+        ext.extra = (
+            UserExtensionInfo(**json.loads(data["_extra"] or "{}"))
+            if "_extra" in data
+            else None
+        )
+        return ext
+
+
+class Extension(NamedTuple):
+    code: str
+    is_valid: bool
+    is_admin_only: bool
+    name: Optional[str] = None
+    short_description: Optional[str] = None
+    tile: Optional[str] = None
+    contributors: Optional[list[str]] = None
+    hidden: bool = False
+    migration_module: Optional[str] = None
+    db_name: Optional[str] = None
+    upgrade_hash: Optional[str] = ""
+
+    @property
+    def module_name(self) -> str:
+        if self.is_upgrade_extension:
+            if settings.has_default_extension_path:
+                return f"lnbits.upgrades.{self.code}-{self.upgrade_hash}"
+            return f"{self.code}-{self.upgrade_hash}"
+
+        if settings.has_default_extension_path:
+            return f"lnbits.extensions.{self.code}"
+        return self.code
+
+    @property
+    def is_upgrade_extension(self) -> bool:
+        return self.upgrade_hash != ""
+
+    @classmethod
+    def from_installable_ext(cls, ext_info: InstallableExtension) -> Extension:
+        return Extension(
+            code=ext_info.id,
+            is_valid=True,
+            is_admin_only=False,  # todo: is admin only
+            name=ext_info.name,
+            upgrade_hash=ext_info.hash if ext_info.module_installed else "",
+        )
+
+    @classmethod
+    def get_valid_extensions(
+        cls, include_deactivated: Optional[bool] = True
+    ) -> list[Extension]:
+        valid_extensions = [
+            extension for extension in cls._extensions() if extension.is_valid
+        ]
+
+        if include_deactivated:
+            return valid_extensions
+
+        if settings.lnbits_extensions_deactivate_all:
+            return []
+
+        return [
+            e
+            for e in valid_extensions
+            if e.code not in settings.lnbits_deactivated_extensions
+        ]
+
+    @classmethod
+    def get_valid_extension(
+        cls, ext_id: str, include_deactivated: Optional[bool] = True
+    ) -> Optional[Extension]:
+        all_extensions = cls.get_valid_extensions(include_deactivated)
+        return next((e for e in all_extensions if e.code == ext_id), None)
+
+    @classmethod
+    def _extensions(cls) -> list[Extension]:
+        p = Path(settings.lnbits_extensions_path, "extensions")
+        Path(p).mkdir(parents=True, exist_ok=True)
+        extension_folders: list[Path] = [f for f in p.iterdir() if f.is_dir()]
+
+        # todo: remove this property somehow, it is too expensive
+        output: list[Extension] = []
+
+        for extension_folder in extension_folders:
+            extension_code = extension_folder.parts[-1]
+            try:
+                with open(extension_folder / "config.json") as json_file:
+                    config = json.load(json_file)
+                is_valid = True
+                is_admin_only = extension_code in settings.lnbits_admin_extensions
+            except Exception:
+                config = {}
+                is_valid = False
+                is_admin_only = False
+
+            output.append(
+                Extension(
+                    extension_code,
+                    is_valid,
+                    is_admin_only,
+                    config.get("name"),
+                    config.get("short_description"),
+                    config.get("tile"),
+                    config.get("contributors"),
+                    config.get("hidden") or False,
+                    config.get("migration_module"),
+                    config.get("db_name"),
+                )
+            )
+
+        return output
+
+
+class ExtensionRelease(BaseModel):
+    name: str
+    version: str
+    archive: str
+    source_repo: str
+    is_github_release: bool = False
+    hash: Optional[str] = None
+    min_lnbits_version: Optional[str] = None
+    is_version_compatible: Optional[bool] = True
+    html_url: Optional[str] = None
+    description: Optional[str] = None
+    warning: Optional[str] = None
+    repo: Optional[str] = None
+    icon: Optional[str] = None
+    details_link: Optional[str] = None
+
+    pay_link: Optional[str] = None
+    cost_sats: Optional[int] = None
+    paid_sats: Optional[int] = 0
+    payment_hash: Optional[str] = None
+
+    @property
+    def archive_url(self) -> str:
+        if not self.pay_link:
+            return self.archive
+        return (
+            f"{self.archive}?version=v{self.version}&payment_hash={self.payment_hash}"
+        )
+
+    async def check_payment_requirements(self):
+        if not self.pay_link:
+            return
+
+        payment_info = await self.fetch_release_payment_info()
+        self.cost_sats = payment_info.amount if payment_info else None
+
+    async def fetch_release_payment_info(
+        self, amount: Optional[int] = None
+    ) -> Optional[ReleasePaymentInfo]:
+        url = f"{self.pay_link}?amount={amount}" if amount else self.pay_link
+        assert url, "Missing URL for payment info."
+        try:
+            async with httpx.AsyncClient() as client:
+                resp = await client.get(url)
+                resp.raise_for_status()
+                return ReleasePaymentInfo(**resp.json())
+        except Exception as e:
+            logger.warning(e)
+            return None
+
+    @classmethod
+    def from_github_release(
+        cls, source_repo: str, r: GitHubRepoRelease
+    ) -> ExtensionRelease:
+        return ExtensionRelease(
+            name=r.name,
+            description=r.name,
+            version=r.tag_name,
+            archive=r.zipball_url,
+            source_repo=source_repo,
+            is_github_release=True,
+            details_link=r.details_link(source_repo),
+            repo=f"https://github.com/{source_repo}",
+            html_url=r.html_url,
+        )
+
+    @classmethod
+    def from_explicit_release(
+        cls, source_repo: str, e: ExplicitRelease
+    ) -> ExtensionRelease:
+        return ExtensionRelease(
+            name=e.name,
+            version=e.version,
+            archive=e.archive,
+            hash=e.hash,
+            source_repo=source_repo,
+            description=e.short_description,
+            min_lnbits_version=e.min_lnbits_version,
+            is_version_compatible=e.is_version_compatible(),
+            warning=e.warning,
+            html_url=e.html_url,
+            details_link=e.details_link,
+            pay_link=e.pay_link,
+            repo=e.repo,
+            icon=e.icon,
+        )
+
+    @classmethod
+    async def get_github_releases(cls, org: str, repo: str) -> list[ExtensionRelease]:
+        try:
+            github_releases = await cls.fetch_github_releases(org, repo)
+            return [
+                ExtensionRelease.from_github_release(f"{org}/{repo}", r)
+                for r in github_releases
+            ]
+        except Exception as e:
+            logger.warning(e)
+            return []
+
+    @classmethod
+    async def fetch_github_releases(
+        cls, org: str, repo: str
+    ) -> list[GitHubRepoRelease]:
+        releases_url = f"https://api.github.com/repos/{org}/{repo}/releases"
+        error_msg = "Cannot fetch extension releases"
+        releases = await github_api_get(releases_url, error_msg)
+        return [GitHubRepoRelease.parse_obj(r) for r in releases]
+
+    @classmethod
+    async def fetch_release_details(cls, details_link: str) -> Optional[dict]:
+
+        try:
+            async with httpx.AsyncClient() as client:
+                resp = await client.get(details_link)
+                resp.raise_for_status()
+                data = resp.json()
+                if "description_md" in data:
+                    resp = await client.get(data["description_md"])
+                    if not resp.is_error:
+                        data["description_md"] = resp.text
+
+                return data
+        except Exception as e:
+            logger.warning(e)
+            return None
+
+
+class InstallableExtension(BaseModel):
+    id: str
+    name: str
+    active: Optional[bool] = False
+    short_description: Optional[str] = None
+    icon: Optional[str] = None
+    dependencies: list[str] = []
+    is_admin_only: bool = False
+    stars: int = 0
+    featured = False
+    latest_release: Optional[ExtensionRelease] = None
+    installed_release: Optional[ExtensionRelease] = None
+    payments: list[ReleasePaymentInfo] = []
+    pay_to_enable: Optional[PayToEnableInfo] = None
+    archive: Optional[str] = None
+
+    @property
+    def hash(self) -> str:
+        if self.installed_release:
+            if self.installed_release.hash:
+                return self.installed_release.hash
+            m = hashlib.sha256()
+            m.update(f"{self.installed_release.archive}".encode())
+            return m.hexdigest()
+        return "not-installed"
+
+    @property
+    def zip_path(self) -> Path:
+        extensions_data_dir = Path(settings.lnbits_data_folder, "zips")
+        Path(extensions_data_dir).mkdir(parents=True, exist_ok=True)
+        return Path(extensions_data_dir, f"{self.id}.zip")
+
+    @property
+    def ext_dir(self) -> Path:
+        return Path(settings.lnbits_extensions_path, "extensions", self.id)
+
+    @property
+    def ext_upgrade_dir(self) -> Path:
+        return Path(
+            settings.lnbits_extensions_path, "upgrades", f"{self.id}-{self.hash}"
+        )
+
+    @property
+    def module_name(self) -> str:
+        if settings.has_default_extension_path:
+            return f"lnbits.extensions.{self.id}"
+        return self.id
+
+    @property
+    def module_installed(self) -> bool:
+        return self.module_name in sys.modules
+
+    @property
+    def has_installed_version(self) -> bool:
+        if not self.ext_dir.is_dir():
+            return False
+        return Path(self.ext_dir, "config.json").is_file()
+
+    @property
+    def installed_version(self) -> str:
+        if self.installed_release:
+            return self.installed_release.version
+        return ""
+
+    @property
+    def requires_payment(self) -> bool:
+        if not self.pay_to_enable:
+            return False
+        return self.pay_to_enable.required is True
+
+    async def download_archive(self):
+        logger.info(f"Downloading extension {self.name} ({self.installed_version}).")
+        ext_zip_file = self.zip_path
+        if ext_zip_file.is_file():
+            os.remove(ext_zip_file)
+        try:
+            assert self.installed_release, "installed_release is none."
+
+            self._restore_payment_info()
+
+            await asyncio.to_thread(
+                download_url, self.installed_release.archive_url, ext_zip_file
+            )
+
+            self._remember_payment_info()
+
+        except Exception as exc:
+            logger.warning(exc)
+            raise AssertionError("Cannot fetch extension archive file") from exc
+
+        archive_hash = file_hash(ext_zip_file)
+        if self.installed_release.hash and self.installed_release.hash != archive_hash:
+            # remove downloaded archive
+            if ext_zip_file.is_file():
+                os.remove(ext_zip_file)
+            raise AssertionError("File hash missmatch. Will not install.")
+
+    def extract_archive(self):
+        logger.info(f"Extracting extension {self.name} ({self.installed_version}).")
+        Path(settings.lnbits_extensions_path, "upgrades").mkdir(
+            parents=True, exist_ok=True
+        )
+
+        tmp_dir = Path(settings.lnbits_data_folder, "unzip-temp", self.hash)
+        shutil.rmtree(tmp_dir, True)
+
+        with zipfile.ZipFile(self.zip_path, "r") as zip_ref:
+            zip_ref.extractall(tmp_dir)
+        generated_dir_name = os.listdir(tmp_dir)[0]
+        shutil.rmtree(self.ext_upgrade_dir, True)
+        shutil.copytree(
+            Path(tmp_dir, generated_dir_name),
+            Path(self.ext_upgrade_dir),
+        )
+        shutil.rmtree(tmp_dir, True)
+
+        # Pre-packed extensions can be upgraded
+        # Mark the extension as installed so we know it is not the pre-packed version
+        with open(Path(self.ext_upgrade_dir, "config.json"), "r+") as json_file:
+            config_json = json.load(json_file)
+
+            self.name = config_json.get("name")
+            self.short_description = config_json.get("short_description")
+
+            if (
+                self.installed_release
+                and self.installed_release.is_github_release
+                and config_json.get("tile")
+            ):
+                self.icon = icon_to_github_url(
+                    self.installed_release.source_repo, config_json.get("tile")
+                )
+
+        shutil.rmtree(self.ext_dir, True)
+        shutil.copytree(Path(self.ext_upgrade_dir), Path(self.ext_dir))
+        logger.success(f"Extension {self.name} ({self.installed_version}) installed.")
+
+    def clean_extension_files(self):
+        # remove downloaded archive
+        if self.zip_path.is_file():
+            os.remove(self.zip_path)
+
+        # remove module from extensions
+        shutil.rmtree(self.ext_dir, True)
+
+        shutil.rmtree(self.ext_upgrade_dir, True)
+
+    def check_latest_version(self, release: Optional[ExtensionRelease]):
+        if not release:
+            return
+        if not self.latest_release:
+            self.latest_release = release
+            return
+        if version_parse(self.latest_release.version) < version_parse(release.version):
+            self.latest_release = release
+
+    def find_existing_payment(
+        self, pay_link: Optional[str]
+    ) -> Optional[ReleasePaymentInfo]:
+        if not pay_link:
+            return None
+        return next(
+            (p for p in self.payments if p.pay_link == pay_link),
+            None,
+        )
+
+    def _restore_payment_info(self):
+        if not self.installed_release:
+            return
+        if not self.installed_release.pay_link:
+            return
+        if self.installed_release.payment_hash:
+            return
+        payment_info = self.find_existing_payment(self.installed_release.pay_link)
+        if payment_info:
+            self.installed_release.payment_hash = payment_info.payment_hash
+
+    def _remember_payment_info(self):
+        if not self.installed_release or not self.installed_release.pay_link:
+            return
+        payment_info = ReleasePaymentInfo(
+            amount=self.installed_release.cost_sats,
+            pay_link=self.installed_release.pay_link,
+            payment_hash=self.installed_release.payment_hash,
+        )
+        self.payments = [
+            p for p in self.payments if p.pay_link != payment_info.pay_link
+        ]
+        self.payments.append(payment_info)
+
+    @classmethod
+    def from_row(cls, data: dict) -> InstallableExtension:
+        meta = json.loads(data["meta"])
+        ext = InstallableExtension(**data)
+        if "installed_release" in meta:
+            ext.installed_release = ExtensionRelease(**meta["installed_release"])
+        if meta.get("pay_to_enable"):
+            ext.pay_to_enable = PayToEnableInfo(**meta["pay_to_enable"])
+        if meta.get("payments"):
+            ext.payments = [ReleasePaymentInfo(**p) for p in meta["payments"]]
+
+        return ext
+
+    @classmethod
+    def from_rows(cls, rows: Optional[list[Any]] = None) -> list[InstallableExtension]:
+        if rows is None:
+            rows = []
+        return [InstallableExtension.from_row(row) for row in rows]
+
+    @classmethod
+    async def from_github_release(
+        cls, github_release: GitHubRelease
+    ) -> Optional[InstallableExtension]:
+        try:
+            repo, latest_release, config = await cls.fetch_github_repo_info(
+                github_release.organisation, github_release.repository
+            )
+            source_repo = f"{github_release.organisation}/{github_release.repository}"
+            return InstallableExtension(
+                id=github_release.id,
+                name=config.name,
+                short_description=config.short_description,
+                stars=int(repo.stargazers_count),
+                icon=icon_to_github_url(
+                    source_repo,
+                    config.tile,
+                ),
+                latest_release=ExtensionRelease.from_github_release(
+                    source_repo, latest_release
+                ),
+            )
+        except Exception as e:
+            logger.warning(e)
+        return None
+
+    @classmethod
+    def from_explicit_release(cls, e: ExplicitRelease) -> InstallableExtension:
+        return InstallableExtension(
+            id=e.id,
+            name=e.name,
+            archive=e.archive,
+            short_description=e.short_description,
+            icon=e.icon,
+            dependencies=e.dependencies,
+        )
+
+    @classmethod
+    async def get_installable_extensions(
+        cls,
+    ) -> list[InstallableExtension]:
+        extension_list: list[InstallableExtension] = []
+        extension_id_list: list[str] = []
+
+        for url in settings.lnbits_extensions_manifests:
+            try:
+                manifest = await cls.fetch_manifest(url)
+
+                for r in manifest.repos:
+                    ext = await InstallableExtension.from_github_release(r)
+                    if not ext:
+                        continue
+                    existing_ext = next(
+                        (ee for ee in extension_list if ee.id == r.id), None
+                    )
+                    if existing_ext:
+                        existing_ext.check_latest_version(ext.latest_release)
+                        continue
+
+                    ext.featured = ext.id in manifest.featured
+                    extension_list += [ext]
+                    extension_id_list += [ext.id]
+
+                for e in manifest.extensions:
+                    release = ExtensionRelease.from_explicit_release(url, e)
+                    existing_ext = next(
+                        (ee for ee in extension_list if ee.id == e.id), None
+                    )
+                    if existing_ext:
+                        existing_ext.check_latest_version(release)
+                        continue
+                    ext = InstallableExtension.from_explicit_release(e)
+                    ext.check_latest_version(release)
+                    ext.featured = ext.id in manifest.featured
+                    extension_list += [ext]
+                    extension_id_list += [e.id]
+            except Exception as e:
+                logger.warning(f"Manifest {url} failed with '{e!s}'")
+
+        return extension_list
+
+    @classmethod
+    async def get_extension_releases(cls, ext_id: str) -> list[ExtensionRelease]:
+        extension_releases: list[ExtensionRelease] = []
+
+        for url in settings.lnbits_extensions_manifests:
+            try:
+                manifest = await cls.fetch_manifest(url)
+                for r in manifest.repos:
+                    if r.id != ext_id:
+                        continue
+                    repo_releases = await ExtensionRelease.get_github_releases(
+                        r.organisation, r.repository
+                    )
+                    extension_releases += repo_releases
+
+                for e in manifest.extensions:
+                    if e.id != ext_id:
+                        continue
+                    explicit_release = ExtensionRelease.from_explicit_release(url, e)
+                    await explicit_release.check_payment_requirements()
+                    extension_releases.append(explicit_release)
+
+            except Exception as e:
+                logger.warning(f"Manifest {url} failed with '{e!s}'")
+
+        return extension_releases
+
+    @classmethod
+    async def get_extension_release(
+        cls, ext_id: str, source_repo: str, archive: str, version: str
+    ) -> Optional[ExtensionRelease]:
+        all_releases: list[ExtensionRelease] = (
+            await InstallableExtension.get_extension_releases(ext_id)
+        )
+        selected_release = [
+            r
+            for r in all_releases
+            if r.archive == archive
+            and r.source_repo == source_repo
+            and r.version == version
+        ]
+
+        return selected_release[0] if len(selected_release) != 0 else None
+
+    @classmethod
+    async def fetch_github_repo_info(
+        cls, org: str, repository: str
+    ) -> tuple[GitHubRepo, GitHubRepoRelease, ExtensionConfig]:
+        repo_url = f"https://api.github.com/repos/{org}/{repository}"
+        error_msg = "Cannot fetch extension repo"
+        repo = await github_api_get(repo_url, error_msg)
+        github_repo = GitHubRepo.parse_obj(repo)
+
+        lates_release_url = (
+            f"https://api.github.com/repos/{org}/{repository}/releases/latest"
+        )
+        error_msg = "Cannot fetch extension releases"
+        latest_release: Any = await github_api_get(lates_release_url, error_msg)
+
+        config_url = f"https://raw.githubusercontent.com/{org}/{repository}/{github_repo.default_branch}/config.json"
+        error_msg = "Cannot fetch config for extension"
+        config = await github_api_get(config_url, error_msg)
+
+        return (
+            github_repo,
+            GitHubRepoRelease.parse_obj(latest_release),
+            ExtensionConfig.parse_obj(config),
+        )
+
+    @classmethod
+    async def fetch_manifest(cls, url) -> Manifest:
+        error_msg = "Cannot fetch extensions manifest"
+        manifest = await github_api_get(url, error_msg)
+        return Manifest.parse_obj(manifest)
+
+
+class CreateExtension(BaseModel):
+    ext_id: str
+    archive: str
+    source_repo: str
+    version: str
+    cost_sats: Optional[int] = 0
+    payment_hash: Optional[str] = None
+
+
+class ExtensionDetailsRequest(BaseModel):
+    ext_id: str
+    source_repo: str
+    version: str

lnbits/core/helpers.py

@@ -1,9 +1,8 @@
 import importlib
 import re
-from typing import Any, Optional
+from typing import Any
 from uuid import UUID
 
-import httpx
 from loguru import logger
 
 from lnbits.core import migrations as core_migrations
@@ -13,11 +12,10 @@ from lnbits.core.crud import (
     update_migration_version,
 )
 from lnbits.core.db import db as core_db
-from lnbits.db import COCKROACH, POSTGRES, SQLITE, Connection
-from lnbits.extension_manager import (
+from lnbits.core.extensions.models import (
     Extension,
-    get_valid_extensions,
 )
+from lnbits.db import COCKROACH, POSTGRES, SQLITE, Connection
 from lnbits.settings import settings
 
 
@@ -55,68 +53,6 @@ async def run_migration(
                         await update_migration_version(conn, db_name, version)
 
 
-async def stop_extension_background_work(
-    ext_id: str, user: str, access_token: Optional[str] = None
-):
-    """
-    Stop background work for extension (like asyncio.Tasks, WebSockets, etc).
-    Extensions SHOULD expose a `api_stop()` function and/or a DELETE enpoint
-    at the root level of their API.
-    """
-    stopped = await _stop_extension_background_work(ext_id)
-
-    if not stopped:
-        # fallback to REST API call
-        await _stop_extension_background_work_via_api(ext_id, user, access_token)
-
-
-async def _stop_extension_background_work(ext_id) -> bool:
-    upgrade_hash = settings.extension_upgrade_hash(ext_id) or ""
-    ext = Extension(ext_id, True, False, upgrade_hash=upgrade_hash)
-
-    try:
-        logger.info(f"Stopping background work for extension '{ext.module_name}'.")
-        old_module = importlib.import_module(ext.module_name)
-
-        # Extensions must expose an `{ext_id}_stop()` function at the module level
-        # The `api_stop()` function is for backwards compatibility (will be deprecated)
-        stop_fns = [f"{ext_id}_stop", "api_stop"]
-        stop_fn_name = next((fn for fn in stop_fns if hasattr(old_module, fn)), None)
-        assert stop_fn_name, "No stop function found for '{ext.module_name}'"
-
-        stop_fn = getattr(old_module, stop_fn_name)
-        if stop_fn:
-            await stop_fn()
-
-        logger.info(f"Stopped background work for extension '{ext.module_name}'.")
-    except Exception as ex:
-        logger.warning(f"Failed to stop background work for '{ext.module_name}'.")
-        logger.warning(ex)
-        return False
-
-    return True
-
-
-async def _stop_extension_background_work_via_api(ext_id, user, access_token):
-    logger.info(
-        f"Stopping background work for extension '{ext_id}' using the REST API."
-    )
-    async with httpx.AsyncClient() as client:
-        try:
-            url = f"http://{settings.host}:{settings.port}/{ext_id}/api/v1?usr={user}"
-            headers = (
-                {"Authorization": "Bearer " + access_token} if access_token else None
-            )
-            resp = await client.delete(url=url, headers=headers)
-            resp.raise_for_status()
-            logger.info(f"Stopped background work for extension '{ext_id}'.")
-        except Exception as ex:
-            logger.warning(
-                f"Failed to stop background work for '{ext_id}' using the REST API."
-            )
-            logger.warning(ex)
-
-
 def to_valid_user_id(user_id: str) -> UUID:
     if len(user_id) < 32:
         raise ValueError("User ID must have at least 128 bits")
@@ -161,7 +97,7 @@ async def migrate_databases():
     await load_disabled_extension_list()
 
     # todo: revisit, use installed extensions
-    for ext in get_valid_extensions(False):
+    for ext in Extension.get_valid_extensions(False):
         current_version = current_versions.get(ext.code, 0)
         try:
             await migrate_extension_database(ext, current_version)

lnbits/core/views/extension_api.py

@@ -1,8 +1,6 @@
-import sys
 from http import HTTPStatus
 from typing import (
     List,
-    Optional,
 )
 
 from bolt11 import decode as bolt11_decode
@@ -13,10 +11,21 @@ from fastapi import (
 )
 from loguru import logger
 
-from lnbits.core.db import core_app_extra
-from lnbits.core.helpers import (
-    migrate_extension_database,
-    stop_extension_background_work,
+from lnbits.core.extensions.extension_manager import (
+    activate_extension,
+    deactivate_extension,
+    install_extension,
+    uninstall_extension,
+)
+from lnbits.core.extensions.models import (
+    CreateExtension,
+    Extension,
+    ExtensionConfig,
+    ExtensionRelease,
+    InstallableExtension,
+    PayToEnableInfo,
+    ReleasePaymentInfo,
+    UserExtensionInfo,
 )
 from lnbits.core.models import (
     SimpleStatus,
@@ -24,36 +33,18 @@ from lnbits.core.models import (
 )
 from lnbits.core.services import check_transaction_status, create_invoice
 from lnbits.decorators import (
-    check_access_token,
     check_admin,
     check_user_exists,
 )
-from lnbits.extension_manager import (
-    CreateExtension,
-    Extension,
-    ExtensionRelease,
-    InstallableExtension,
-    PayToEnableInfo,
-    ReleasePaymentInfo,
-    UserExtensionInfo,
-    fetch_github_release_config,
-    fetch_release_details,
-    fetch_release_payment_info,
-    get_valid_extensions,
-)
-from lnbits.settings import settings
 
 from ..crud import (
-    add_installed_extension,
     delete_dbversion,
-    delete_installed_extension,
     drop_extension_db,
     get_dbversions,
     get_installed_extension,
     get_installed_extensions,
     get_user_extension,
     update_extension_pay_to_enable,
-    update_installed_extension_state,
     update_user_extension,
     update_user_extension_extra,
 )
@@ -64,12 +55,8 @@ extension_router = APIRouter(
 )
 
 
-@extension_router.post("")
-async def api_install_extension(
-    data: CreateExtension,
-    user: User = Depends(check_admin),
-    access_token: Optional[str] = Depends(check_access_token),
-):
+@extension_router.post("", dependencies=[Depends(check_admin)])
+async def api_install_extension(data: CreateExtension):
     release = await InstallableExtension.get_extension_release(
         data.ext_id, data.source_repo, data.archive, data.version
     )
@@ -89,43 +76,36 @@ async def api_install_extension(
     )
 
     try:
-        installed_ext = await get_installed_extension(data.ext_id)
-        ext_info.payments = installed_ext.payments if installed_ext else []
+        extension = await install_extension(ext_info)
 
-        await ext_info.download_archive()
-
-        ext_info.extract_archive()
-
-        extension = Extension.from_installable_ext(ext_info)
-
-        db_version = (await get_dbversions()).get(data.ext_id, 0)
-        await migrate_extension_database(extension, db_version)
-
-        ext_info.active = True
-        await add_installed_extension(ext_info)
-
-        if extension.is_upgrade_extension:
-            # call stop while the old routes are still active
-            await stop_extension_background_work(data.ext_id, user.id, access_token)
-
-        # mount routes for the new version
-        core_app_extra.register_new_ext_routes(extension)
-
-        ext_info.notify_upgrade(extension.upgrade_hash)
-        settings.lnbits_deactivated_extensions.discard(data.ext_id)
-
-        return extension
-    except AssertionError as exc:
-        raise HTTPException(HTTPStatus.BAD_REQUEST, str(exc)) from exc
     except Exception as exc:
         logger.warning(exc)
         ext_info.clean_extension_files()
+        detail = (
+            str(exc)
+            if isinstance(exc, AssertionError)
+            else f"Failed to install extension '{ext_info.id}'."
+            f"({ext_info.installed_version})."
+        )
         raise HTTPException(
             status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
-            detail=(
-                f"Failed to install extension {ext_info.id} "
-                f"({ext_info.installed_version})."
-            ),
+            detail=detail,
+        ) from exc
+
+    try:
+        await activate_extension(extension)
+        return extension
+    except Exception as exc:
+        logger.warning(exc)
+        await deactivate_extension(extension.code)
+        detail = (
+            str(exc)
+            if isinstance(exc, AssertionError)
+            else f"Extension `{extension.code}` installed, but activation failed."
+        )
+        raise HTTPException(
+            status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+            detail=detail,
         ) from exc
 
 
@@ -143,7 +123,7 @@ async def api_extension_details(
         )
         assert release, "Details not found for release"
 
-        release_details = await fetch_release_details(details_link)
+        release_details = await ExtensionRelease.fetch_release_details(details_link)
         assert release_details, "Cannot fetch details for release"
         release_details["icon"] = release.icon
         release_details["repo"] = release.repo
@@ -186,7 +166,7 @@ async def api_update_pay_to_enable(
 async def api_enable_extension(
     ext_id: str, user: User = Depends(check_user_exists)
 ) -> SimpleStatus:
-    if ext_id not in [e.code for e in get_valid_extensions()]:
+    if ext_id not in [e.code for e in Extension.get_valid_extensions()]:
         raise HTTPException(
             HTTPStatus.NOT_FOUND, f"Extension '{ext_id}' doesn't exist."
         )
@@ -249,7 +229,7 @@ async def api_enable_extension(
 async def api_disable_extension(
     ext_id: str, user: User = Depends(check_user_exists)
 ) -> SimpleStatus:
-    if ext_id not in [e.code for e in get_valid_extensions()]:
+    if ext_id not in [e.code for e in Extension.get_valid_extensions()]:
         raise HTTPException(
             HTTPStatus.BAD_REQUEST, f"Extension '{ext_id}' doesn't exist."
         )
@@ -270,20 +250,14 @@ async def api_activate_extension(ext_id: str) -> SimpleStatus:
     try:
         logger.info(f"Activating extension: '{ext_id}'.")
 
-        all_extensions = get_valid_extensions()
-        ext = next((e for e in all_extensions if e.code == ext_id), None)
+        ext = Extension.get_valid_extension(ext_id)
         assert ext, f"Extension '{ext_id}' doesn't exist."
-        # if extension never loaded (was deactivated on server startup)
-        if ext_id not in sys.modules.keys():
-            # run extension start-up routine
-            core_app_extra.register_new_ext_routes(ext)
 
-        settings.lnbits_deactivated_extensions.discard(ext_id)
-
-        await update_installed_extension_state(ext_id=ext_id, active=True)
+        await activate_extension(ext)
         return SimpleStatus(success=True, message=f"Extension '{ext_id}' activated.")
     except Exception as exc:
         logger.warning(exc)
+        await deactivate_extension(ext_id)
         raise HTTPException(
             status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
             detail=(f"Failed to activate '{ext_id}'."),
@@ -295,13 +269,10 @@ async def api_deactivate_extension(ext_id: str) -> SimpleStatus:
     try:
         logger.info(f"Deactivating extension: '{ext_id}'.")
 
-        all_extensions = get_valid_extensions()
-        ext = next((e for e in all_extensions if e.code == ext_id), None)
+        ext = Extension.get_valid_extension(ext_id)
         assert ext, f"Extension '{ext_id}' doesn't exist."
 
-        settings.lnbits_deactivated_extensions.add(ext_id)
-
-        await update_installed_extension_state(ext_id=ext_id, active=False)
+        await deactivate_extension(ext_id)
         return SimpleStatus(success=True, message=f"Extension '{ext_id}' deactivated.")
     except Exception as exc:
         logger.warning(exc)
@@ -311,23 +282,19 @@ async def api_deactivate_extension(ext_id: str) -> SimpleStatus:
         ) from exc
 
 
-@extension_router.delete("/{ext_id}")
-async def api_uninstall_extension(
-    ext_id: str,
-    user: User = Depends(check_admin),
-    access_token: Optional[str] = Depends(check_access_token),
-) -> SimpleStatus:
-    installed_extensions = await get_installed_extensions()
+@extension_router.delete("/{ext_id}", dependencies=[Depends(check_admin)])
+async def api_uninstall_extension(ext_id: str) -> SimpleStatus:
 
-    extensions = [e for e in installed_extensions if e.id == ext_id]
-    if len(extensions) == 0:
+    extension = await get_installed_extension(ext_id)
+    if not extension:
         raise HTTPException(
             status_code=HTTPStatus.NOT_FOUND,
             detail=f"Unknown extension id: {ext_id}",
         )
 
+    installed_extensions = await get_installed_extensions()
     # check that other extensions do not depend on this one
-    for valid_ext_id in [ext.code for ext in get_valid_extensions()]:
+    for valid_ext_id in [ext.code for ext in Extension.get_valid_extensions()]:
         installed_ext = next(
             (ext for ext in installed_extensions if ext.id == valid_ext_id), None
         )
@@ -341,14 +308,7 @@ async def api_uninstall_extension(
             )
 
     try:
-        # call stop while the old routes are still active
-        await stop_extension_background_work(ext_id, user.id, access_token)
-
-        settings.lnbits_deactivated_extensions.add(ext_id)
-
-        for ext_info in extensions:
-            ext_info.clean_extension_files()
-            await delete_installed_extension(ext_id=ext_info.id)
+        await uninstall_extension(ext_id)
 
         logger.success(f"Extension '{ext_id}' uninstalled.")
         return SimpleStatus(success=True, message=f"Extension '{ext_id}' uninstalled.")
@@ -397,9 +357,8 @@ async def get_pay_to_install_invoice(
         assert release, "Release not found."
         assert release.pay_link, "Pay link not found for release."
 
-        payment_info = await fetch_release_payment_info(
-            release.pay_link, data.cost_sats
-        )
+        payment_info = await release.fetch_release_payment_info(data.cost_sats)
+
         assert payment_info and payment_info.payment_request, "Cannot request invoice."
         invoice = bolt11_decode(payment_info.payment_request)
 
@@ -474,7 +433,7 @@ async def get_pay_to_enable_invoice(
 )
 async def get_extension_release(org: str, repo: str, tag_name: str):
     try:
-        config = await fetch_github_release_config(org, repo, tag_name)
+        config = await ExtensionConfig.fetch_github_release_config(org, repo, tag_name)
         if not config:
             return {}
 

lnbits/core/views/generic.py

@@ -12,6 +12,7 @@ from lnurl import decode as lnurl_decode
 from loguru import logger
 from pydantic.types import UUID4
 
+from lnbits.core.extensions.models import Extension, InstallableExtension
 from lnbits.core.helpers import to_valid_user_id
 from lnbits.core.models import User
 from lnbits.core.services import create_invoice
@@ -20,7 +21,6 @@ from lnbits.helpers import template_renderer
 from lnbits.settings import settings
 from lnbits.wallets import get_funding_source
 
-from ...extension_manager import InstallableExtension, get_valid_extensions
 from ...utils.exchange_rates import allowed_currencies, currencies
 from ..crud import (
     create_account,
@@ -104,7 +104,7 @@ async def extensions(request: Request, user: User = Depends(check_user_exists)):
         installed_exts_ids = []
 
     try:
-        all_ext_ids = [ext.code for ext in get_valid_extensions()]
+        all_ext_ids = [ext.code for ext in Extension.get_valid_extensions()]
         inactive_extensions = [
             e.id for e in await get_installed_extensions(active=False)
         ]