feat(storage): S3-compatible storage adapter + storage_uri rename

Add S3FileStorage adapter (any S3-compatible backend: AWS, MinIO, Garage)
alongside the local adapter, selected via STORAGE_BACKEND. Proxied range
streaming via get_object+Range; as_local_path downloads to a tempfile for
ffmpeg/fpcalc. Rename track.file_path -> storage_uri across domain entity,
ORM, repositories, port, and services, with an Alembic migration. Adds
mocked S3 unit tests.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Senko-san
2026-06-08 17:11:35 +03:00
parent a8348e145a
commit 5c5df5d3cc
15 changed files with 1377 additions and 498 deletions
+1 -1
View File
@@ -40,7 +40,7 @@ class TrackModel(UUIDPrimaryKeyMixin, TimestampMixin, Base):
year: Mapped[int | None] = mapped_column(Integer, nullable=True)
# -- file (original, stored as-is) -----------------------------------
file_path: Mapped[str] = mapped_column(String(2048), nullable=False)
storage_uri: Mapped[str] = mapped_column(String(2048), nullable=False)
file_format: Mapped[str] = mapped_column(String(32), nullable=False)
file_size: Mapped[int] = mapped_column(Integer, nullable=False)
bitrate: Mapped[int | None] = mapped_column(Integer, nullable=True)
@@ -30,7 +30,7 @@ def _track_to_entity(row: TrackModel) -> Track:
title=row.title,
artist_id=row.artist_id,
album_id=row.album_id,
file_path=row.file_path,
storage_uri=row.storage_uri,
file_format=row.file_format,
file_size=row.file_size,
source=row.source,
@@ -29,7 +29,7 @@ def _track_to_entity(row: TrackModel) -> Track:
title=row.title,
artist_id=row.artist_id,
album_id=row.album_id,
file_path=row.file_path,
storage_uri=row.storage_uri,
file_format=row.file_format,
file_size=row.file_size,
source=row.source,
@@ -17,7 +17,7 @@ def _to_entity(row: TrackModel) -> Track:
title=row.title,
artist_id=row.artist_id,
album_id=row.album_id,
file_path=row.file_path,
storage_uri=row.storage_uri,
file_format=row.file_format,
file_size=row.file_size,
source=row.source,
@@ -56,7 +56,7 @@ class SqlAlchemyTrackRepository:
id: uuid.UUID,
title: str,
artist_id: uuid.UUID,
file_path: str,
storage_uri: str,
file_format: str,
file_size: int,
source: str,
@@ -68,7 +68,7 @@ class SqlAlchemyTrackRepository:
id=id,
title=title,
artist_id=artist_id,
file_path=file_path,
storage_uri=storage_uri,
file_format=file_format,
file_size=file_size,
source=source,
+20 -5
View File
@@ -1,16 +1,31 @@
"""File storage provider — singleton factory."""
"""File storage provider — singleton factory wired from config."""
from app.core.config import get_settings
from app.domain.ports import FileStorage
from app.infrastructure.storage.local import LocalFileStorage
from app.infrastructure.storage.s3 import S3FileStorage
_storage: LocalFileStorage | None = None
_storage: FileStorage | None = None
def get_file_storage() -> LocalFileStorage:
def get_file_storage() -> FileStorage:
global _storage
if _storage is None:
settings = get_settings()
if settings.storage_backend == "s3":
raise NotImplementedError("S3 storage not yet implemented.")
_storage = LocalFileStorage(settings.media_path)
if not settings.s3_bucket:
raise RuntimeError("S3_BUCKET must be set when STORAGE_BACKEND=s3")
_storage = S3FileStorage(
settings.s3_bucket,
endpoint_url=settings.s3_endpoint_url,
region_name=settings.s3_region,
access_key=settings.s3_access_key.get_secret_value()
if settings.s3_access_key
else None,
secret_key=settings.s3_secret_key.get_secret_value()
if settings.s3_secret_key
else None,
)
else:
_storage = LocalFileStorage(settings.media_path)
return _storage
+157
View File
@@ -0,0 +1,157 @@
"""S3FileStorage — stores files in any S3-compatible object store."""
import tempfile
from collections.abc import AsyncGenerator, AsyncIterator
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from pathlib import Path
from typing import Any
import aioboto3
import anyio
from botocore.exceptions import ClientError
from app.domain.entities.storage import ObjectStat
from app.domain.errors import StorageError
_EXT_CONTENT_TYPE: dict[str, str] = {
"mp3": "audio/mpeg",
"flac": "audio/flac",
"m4a": "audio/mp4",
"aac": "audio/aac",
"ogg": "audio/ogg",
"opus": "audio/ogg",
"wav": "audio/wav",
"wma": "audio/x-ms-wma",
"aiff": "audio/aiff",
"aif": "audio/aiff",
}
def _not_found(key: str, exc: Exception) -> StorageError:
err = StorageError(f"Object not found: {key}")
err.__cause__ = exc
return err
def _is_404(exc: ClientError) -> bool:
return exc.response["Error"]["Code"] in ("404", "NoSuchKey")
class S3FileStorage:
def __init__(
self,
bucket: str,
*,
endpoint_url: str | None = None,
region_name: str | None = None,
access_key: str | None = None,
secret_key: str | None = None,
) -> None:
self._bucket = bucket
self._endpoint_url = endpoint_url
self._session: Any = aioboto3.Session(
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
region_name=region_name,
)
def _client(self) -> Any:
return self._session.client("s3", endpoint_url=self._endpoint_url)
async def save_file(self, key: str, src_path: Path) -> int:
async with await anyio.open_file(src_path, "rb") as f:
content: bytes = await f.read()
async with self._client() as s3:
await s3.put_object(Bucket=self._bucket, Key=key, Body=content)
return len(content)
async def open_range(
self, key: str, start: int, end: int | None
) -> tuple[AsyncIterator[bytes], int]:
async with self._client() as s3:
try:
head = await s3.head_object(Bucket=self._bucket, Key=key)
except ClientError as exc:
if _is_404(exc):
raise _not_found(key, exc) from exc
raise StorageError(str(exc)) from exc
total_size: int = head["ContentLength"]
range_header = f"bytes={start}-{end}" if end is not None else f"bytes={start}-"
_bucket = self._bucket
_key = key
async def _stream() -> AsyncGenerator[bytes]:
async with self._client() as s3:
try:
resp = await s3.get_object(
Bucket=_bucket, Key=_key, Range=range_header
)
except ClientError as exc:
raise StorageError(str(exc)) from exc
body = resp["Body"]
while True:
chunk: bytes = await body.read(65536)
if not chunk:
break
yield chunk
return _stream(), total_size
async def stat(self, key: str) -> ObjectStat:
async with self._client() as s3:
try:
head = await s3.head_object(Bucket=self._bucket, Key=key)
except ClientError as exc:
if _is_404(exc):
raise _not_found(key, exc) from exc
raise StorageError(str(exc)) from exc
ext = Path(key).suffix.lower().lstrip(".")
return ObjectStat(
size=head["ContentLength"],
content_type=head.get("ContentType") or _EXT_CONTENT_TYPE.get(ext),
)
async def exists(self, key: str) -> bool:
async with self._client() as s3:
try:
await s3.head_object(Bucket=self._bucket, Key=key)
return True
except ClientError as exc:
if _is_404(exc):
return False
raise StorageError(str(exc)) from exc
async def delete(self, key: str) -> None:
async with self._client() as s3:
try:
await s3.delete_object(Bucket=self._bucket, Key=key)
except ClientError as exc:
raise StorageError(str(exc)) from exc
def as_local_path(self, key: str) -> AbstractAsyncContextManager[Path]:
return self._as_local_path_cm(key)
@asynccontextmanager
async def _as_local_path_cm(self, key: str) -> AsyncGenerator[Path]:
suffix = Path(key).suffix
with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as f:
tmp_path = Path(f.name)
try:
async with self._client() as s3:
try:
resp = await s3.get_object(Bucket=self._bucket, Key=key)
except ClientError as exc:
if _is_404(exc):
raise _not_found(key, exc) from exc
raise StorageError(str(exc)) from exc
async with await anyio.open_file(tmp_path, "wb") as out:
body = resp["Body"]
while True:
chunk: bytes = await body.read(65536)
if not chunk:
break
await out.write(chunk)
yield tmp_path
finally:
await anyio.Path(tmp_path).unlink(missing_ok=True)