feat(sources): local_folder source backend + import pipeline
Docker Build & Publish / build (push) Has been cancelled
Docker Build & Publish / push (push) Has been cancelled
Docker Build & Publish / Prune old image versions (push) Has been cancelled

First ingest path beyond manual upload (plan §1C). Source abstraction +
the first concrete backend, so a homelab can index an existing library.

- domain: SourceBackend/IndexableSource ports + SourceInfo/SourceFile shapes
- infrastructure/sources: LocalFolderSource (walks a mounted dir, idempotent
  source_id = relative path) + registry built from settings
- application: LibraryImportService — batch sibling of UploadService; dedup on
  (source, source_id), copy into storage, minimal track (metadata_status=pending,
  enrichment fills the rest in 1D), per-file failures isolated
- workers: scan_local_folder arq task (registered) + enqueue helper (503 if
  Redis down)
- api: GET /sources, POST /sources/{source}/scan (admin, enqueues), /health
- config: LOCAL_MEDIA_IMPORT_PATH; README + .env.example documented
- tests: scanner, registry, import service (fakes) + DB-gated sources API path

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Senko-san
2026-06-08 20:02:09 +03:00
parent 551afbab13
commit 48e3418c7f
19 changed files with 800 additions and 11 deletions
+9
View File
@@ -35,6 +35,7 @@ from app.infrastructure.db.repositories import (
SqlAlchemyTrackRepository,
SqlAlchemyUserRepository,
)
from app.infrastructure.sources.registry import SourceRegistry, build_source_registry
from app.infrastructure.storage.provider import get_file_storage
@@ -70,6 +71,14 @@ def get_subsonic_cipher() -> SubsonicCipher:
return SubsonicPasswordCipher(get_settings().subsonic_secret_key.get_secret_value())
@lru_cache
def get_source_registry() -> SourceRegistry:
return build_source_registry(get_settings())
SourceRegistryDep = Annotated[SourceRegistry, Depends(get_source_registry)]
# -- request-scoped services ---------------------------------------------------
def get_auth_service(session: SessionDep) -> AuthService:
return AuthService(
+29
View File
@@ -0,0 +1,29 @@
"""Schemas for the source endpoints."""
from pydantic import BaseModel
from app.domain.sources import SourceInfo
class SourceInfoOut(BaseModel):
name: str
label: str
kind: str
available: bool
@classmethod
def from_entity(cls, info: SourceInfo) -> SourceInfoOut:
return cls(name=info.name, label=info.label, kind=info.kind, available=info.available)
class ScanResponse(BaseModel):
"""Result of enqueuing a source scan."""
source: str
job_id: str
status: str = "queued"
class SourceHealthOut(BaseModel):
name: str
available: bool
+30 -5
View File
@@ -1,19 +1,44 @@
"""External source endpoints (yt-dlp etc.)."""
"""External source endpoints: enumerate sources and trigger imports.
Listing/health are read-only (any authenticated user). Scanning a source is an
admin action and runs in a worker — the endpoint only enqueues it.
"""
from typing import Any
from fastapi import APIRouter
from app.api.deps import CurrentUser, SourceRegistryDep, SuperUser
from app.api.schemas.source import ScanResponse, SourceHealthOut, SourceInfoOut
from app.domain.errors import DependencyUnavailableError
from app.workers.queue import enqueue
router = APIRouter(prefix="/sources", tags=["sources"])
@router.get("")
async def list_sources() -> Any: ...
async def list_sources(_: CurrentUser, registry: SourceRegistryDep) -> list[SourceInfoOut]:
return [SourceInfoOut.from_entity(info) for info in registry.infos()]
@router.get("/{source}/search")
async def search_source(source: str) -> Any: ...
@router.post("/{source}/scan")
async def scan_source(source: str, user: SuperUser, registry: SourceRegistryDep) -> ScanResponse:
backend = registry.indexable(source) # 404 if unknown, 422 if not indexable
if not backend.is_available():
raise DependencyUnavailableError(f"Source {source!r} is not available.")
job_id = await enqueue("scan_local_folder", source=source, added_by=str(user.id))
return ScanResponse(source=source, job_id=job_id)
@router.get("/{source}/health")
async def source_health(source: str) -> Any: ...
async def source_health(
source: str, _: CurrentUser, registry: SourceRegistryDep
) -> SourceHealthOut:
backend = registry.get(source) # 404 if unknown
return SourceHealthOut(name=backend.name, available=backend.is_available())
@router.get("/{source}/search")
async def search_source(source: str, _: CurrentUser) -> Any:
# Search is for fetch-style sources (youtube, …) — not yet implemented.
...