feat(sources): local_folder source backend + import pipeline
First ingest path beyond manual upload (plan §1C). Source abstraction +
the first concrete backend, so a homelab can index an existing library.
- domain: SourceBackend/IndexableSource ports + SourceInfo/SourceFile shapes
- infrastructure/sources: LocalFolderSource (walks a mounted dir, idempotent
source_id = relative path) + registry built from settings
- application: LibraryImportService — batch sibling of UploadService; dedup on
(source, source_id), copy into storage, minimal track (metadata_status=pending,
enrichment fills the rest in 1D), per-file failures isolated
- workers: scan_local_folder arq task (registered) + enqueue helper (503 if
Redis down)
- api: GET /sources, POST /sources/{source}/scan (admin, enqueues), /health
- config: LOCAL_MEDIA_IMPORT_PATH; README + .env.example documented
- tests: scanner, registry, import service (fakes) + DB-gated sources API path
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,96 @@
|
||||
"""LibraryImportService — imports files discovered by an indexable source.
|
||||
|
||||
Batch sibling of :class:`UploadService`: for each discovered file it dedups on
|
||||
``(source, source_id)``, copies the file into managed storage, creates a minimal
|
||||
track (artist ``Unknown Artist``, ``metadata_status=pending``), and leaves the
|
||||
rest to enrichment (plan §6.2). Per-file failures are isolated — one bad file
|
||||
must not abort the whole scan (graceful degradation).
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.core.logging import get_logger
|
||||
from app.domain.ports import ArtistRepository, FileStorage, IndexableSource, TrackRepository
|
||||
from app.domain.sources import SourceFile
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
_UNKNOWN_ARTIST = "Unknown Artist"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImportSummary:
|
||||
source: str
|
||||
seen: int
|
||||
imported: int
|
||||
skipped: int
|
||||
failed: int
|
||||
|
||||
|
||||
class LibraryImportService:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
tracks: TrackRepository,
|
||||
artists: ArtistRepository,
|
||||
storage: FileStorage,
|
||||
) -> None:
|
||||
self._tracks = tracks
|
||||
self._artists = artists
|
||||
self._storage = storage
|
||||
|
||||
async def scan_and_import(
|
||||
self, source: IndexableSource, *, added_by: uuid.UUID | None
|
||||
) -> ImportSummary:
|
||||
seen = imported = skipped = failed = 0
|
||||
for file in source.scan():
|
||||
seen += 1
|
||||
try:
|
||||
existing = await self._tracks.get_by_source(source.name, file.source_id)
|
||||
if existing is not None:
|
||||
skipped += 1
|
||||
continue
|
||||
await self._import_one(source.name, file, added_by)
|
||||
imported += 1
|
||||
except Exception:
|
||||
failed += 1
|
||||
log.warning("import_file_failed", source=source.name, source_id=file.source_id)
|
||||
summary = ImportSummary(
|
||||
source=source.name, seen=seen, imported=imported, skipped=skipped, failed=failed
|
||||
)
|
||||
log.info(
|
||||
"import_complete",
|
||||
source=summary.source,
|
||||
seen=summary.seen,
|
||||
imported=summary.imported,
|
||||
skipped=summary.skipped,
|
||||
failed=summary.failed,
|
||||
)
|
||||
return summary
|
||||
|
||||
async def _import_one(
|
||||
self, source_name: str, file: SourceFile, added_by: uuid.UUID | None
|
||||
) -> None:
|
||||
track_id = uuid.uuid4()
|
||||
key = f"tracks/{str(track_id)[:2]}/{track_id}.{file.file_format}"
|
||||
await self._storage.save_file(key, file.path)
|
||||
try:
|
||||
artist = await self._artists.get_or_create(_UNKNOWN_ARTIST)
|
||||
await self._tracks.add(
|
||||
id=track_id,
|
||||
title=file.suggested_title,
|
||||
artist_id=artist.id,
|
||||
storage_uri=key,
|
||||
file_format=file.file_format,
|
||||
file_size=file.file_size,
|
||||
source=source_name,
|
||||
source_id=file.source_id,
|
||||
metadata_status="pending",
|
||||
added_by=added_by,
|
||||
)
|
||||
except Exception:
|
||||
with contextlib.suppress(Exception):
|
||||
await self._storage.delete(key)
|
||||
raise
|
||||
Reference in New Issue
Block a user