feat(sources): local_folder source backend + import pipeline
First ingest path beyond manual upload (plan §1C). Source abstraction +
the first concrete backend, so a homelab can index an existing library.
- domain: SourceBackend/IndexableSource ports + SourceInfo/SourceFile shapes
- infrastructure/sources: LocalFolderSource (walks a mounted dir, idempotent
source_id = relative path) + registry built from settings
- application: LibraryImportService — batch sibling of UploadService; dedup on
(source, source_id), copy into storage, minimal track (metadata_status=pending,
enrichment fills the rest in 1D), per-file failures isolated
- workers: scan_local_folder arq task (registered) + enqueue helper (503 if
Redis down)
- api: GET /sources, POST /sources/{source}/scan (admin, enqueues), /health
- config: LOCAL_MEDIA_IMPORT_PATH; README + .env.example documented
- tests: scanner, registry, import service (fakes) + DB-gated sources API path
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,151 @@
|
||||
"""Integration tests for sources: enumeration + the real import path.
|
||||
|
||||
Requires a reachable Postgres; skips otherwise. The scan worker task is invoked
|
||||
directly (no Redis needed) so the full DB + storage import path is covered.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from collections.abc import AsyncIterator
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from app.core.config import get_settings
|
||||
from app.infrastructure.db import Base, dispose_engine, get_engine, session_scope
|
||||
from app.infrastructure.db.repositories import (
|
||||
SqlAlchemyRefreshTokenRepository,
|
||||
SqlAlchemyUserRepository,
|
||||
)
|
||||
from asgi_lifespan import LifespanManager
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
_db_reachable_cache: bool | None = None
|
||||
|
||||
|
||||
async def _db_reachable() -> bool:
|
||||
global _db_reachable_cache
|
||||
if _db_reachable_cache is not None:
|
||||
return _db_reachable_cache
|
||||
from sqlalchemy import text
|
||||
|
||||
try:
|
||||
async with asyncio.timeout(3):
|
||||
async with get_engine().connect() as conn:
|
||||
await conn.execute(text("SELECT 1"))
|
||||
_db_reachable_cache = True
|
||||
except Exception:
|
||||
_db_reachable_cache = False
|
||||
return _db_reachable_cache
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def api(tmp_path: Path) -> AsyncIterator[AsyncClient]:
|
||||
if not await _db_reachable():
|
||||
pytest.skip("Postgres not reachable — integration test skipped.")
|
||||
|
||||
media = tmp_path / "media"
|
||||
music = tmp_path / "music"
|
||||
media.mkdir()
|
||||
music.mkdir()
|
||||
# Two audio files (+ a non-audio file that must be ignored) in a subfolder.
|
||||
(music / "one.mp3").write_bytes(b"first track bytes" * 8)
|
||||
(music / "artist").mkdir()
|
||||
(music / "artist" / "two.flac").write_bytes(b"second track bytes" * 8)
|
||||
(music / "cover.txt").write_bytes(b"not audio")
|
||||
|
||||
os.environ["MEDIA_PATH"] = str(media)
|
||||
os.environ["LOCAL_MEDIA_IMPORT_PATH"] = str(music)
|
||||
get_settings.cache_clear()
|
||||
|
||||
import app.infrastructure.storage.provider as _storage_provider
|
||||
|
||||
_storage_provider._storage = None
|
||||
|
||||
try:
|
||||
async with get_engine().begin() as conn:
|
||||
await conn.run_sync(Base.metadata.drop_all)
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
from app.application.user_service import UserService
|
||||
from app.core.security import Argon2PasswordHasher
|
||||
|
||||
async with session_scope() as session:
|
||||
await UserService(
|
||||
users=SqlAlchemyUserRepository(session),
|
||||
refresh_tokens=SqlAlchemyRefreshTokenRepository(session),
|
||||
hasher=Argon2PasswordHasher(),
|
||||
).create_user(username="admin", password="adminpass1", is_superuser=True)
|
||||
|
||||
from app.main import create_app
|
||||
|
||||
app = create_app()
|
||||
async with LifespanManager(app):
|
||||
transport = ASGITransport(app=app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
yield client
|
||||
|
||||
async with get_engine().begin() as conn:
|
||||
await conn.run_sync(Base.metadata.drop_all)
|
||||
await dispose_engine()
|
||||
finally:
|
||||
_storage_provider._storage = None
|
||||
os.environ.pop("MEDIA_PATH", None)
|
||||
os.environ.pop("LOCAL_MEDIA_IMPORT_PATH", None)
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
async def _login(api: AsyncClient) -> str:
|
||||
resp = await api.post(
|
||||
"/api/v1/auth/login", json={"username": "admin", "password": "adminpass1"}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
return str(resp.json()["access_token"])
|
||||
|
||||
|
||||
async def test_list_sources_includes_local(api: AsyncClient) -> None:
|
||||
token = await _login(api)
|
||||
resp = await api.get("/api/v1/sources", headers={"Authorization": f"Bearer {token}"})
|
||||
assert resp.status_code == 200
|
||||
sources = {s["name"]: s for s in resp.json()}
|
||||
assert "local" in sources
|
||||
assert sources["local"]["available"] is True
|
||||
assert sources["local"]["kind"] == "indexable"
|
||||
|
||||
|
||||
async def test_local_import_creates_streamable_tracks(api: AsyncClient) -> None:
|
||||
token = await _login(api)
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
# Run the worker task directly (bypasses Redis); it imports against the DB.
|
||||
from app.workers.tasks.import_task import scan_local_folder
|
||||
|
||||
summary = await scan_local_folder({}, source="local", added_by=None)
|
||||
assert summary["seen"] == 2
|
||||
assert summary["imported"] == 2
|
||||
assert summary["failed"] == 0
|
||||
|
||||
# A second run is idempotent — everything already indexed.
|
||||
again = await scan_local_folder({}, source="local", added_by=None)
|
||||
assert again["imported"] == 0
|
||||
assert again["skipped"] == 2
|
||||
|
||||
listing = await api.get("/api/v1/tracks", headers=headers)
|
||||
assert listing.status_code == 200
|
||||
items = listing.json()["items"]
|
||||
assert len(items) == 2
|
||||
titles = {t["title"] for t in items}
|
||||
assert titles == {"one", "two"}
|
||||
|
||||
# And the imported file actually streams back.
|
||||
track_id = items[0]["id"]
|
||||
stream = await api.get(f"/api/v1/stream/{track_id}", headers=headers)
|
||||
assert stream.status_code == 200
|
||||
assert len(stream.content) > 0
|
||||
|
||||
|
||||
async def test_scan_requires_admin(api: AsyncClient) -> None:
|
||||
# The scan endpoint enqueues to Redis; here we only assert it's admin-gated.
|
||||
resp = await api.post("/api/v1/sources/local/scan")
|
||||
assert resp.status_code == 401
|
||||
Reference in New Issue
Block a user