feat(storage): S3-compatible storage adapter + storage_uri rename
Add S3FileStorage adapter (any S3-compatible backend: AWS, MinIO, Garage) alongside the local adapter, selected via STORAGE_BACKEND. Proxied range streaming via get_object+Range; as_local_path downloads to a tempfile for ffmpeg/fpcalc. Rename track.file_path -> storage_uri across domain entity, ORM, repositories, port, and services, with an Alembic migration. Adds mocked S3 unit tests. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Senko-san
@@ -0,0 +1,254 @@
|
||||
"""Unit tests for S3FileStorage — all S3 calls are mocked."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from app.domain.errors import StorageError
|
||||
from app.infrastructure.storage.s3 import S3FileStorage
|
||||
|
||||
|
||||
def _make_storage(**kwargs: Any) -> S3FileStorage:
|
||||
return S3FileStorage("test-bucket", **kwargs)
|
||||
|
||||
|
||||
def _client_error(code: str) -> Exception:
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
return ClientError({"Error": {"Code": code, "Message": code}}, "op")
|
||||
|
||||
|
||||
class _FakeBody:
|
||||
"""Async-iterable body that yields chunks from a bytes buffer."""
|
||||
|
||||
def __init__(self, data: bytes, chunk_size: int = 65536) -> None:
|
||||
self._buf = io.BytesIO(data)
|
||||
self._chunk_size = chunk_size
|
||||
|
||||
async def read(self, size: int = -1) -> bytes:
|
||||
return self._buf.read(size)
|
||||
|
||||
|
||||
def _make_client_ctx(s3_mock: Any) -> Any:
|
||||
ctx = MagicMock()
|
||||
ctx.__aenter__ = AsyncMock(return_value=s3_mock)
|
||||
ctx.__aexit__ = AsyncMock(return_value=False)
|
||||
return ctx
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def storage() -> S3FileStorage:
|
||||
return _make_storage()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# save_file
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_save_file_calls_put_object(tmp_path: Path, storage: S3FileStorage) -> None:
|
||||
src = tmp_path / "track.mp3"
|
||||
src.write_bytes(b"audio bytes")
|
||||
|
||||
s3 = AsyncMock()
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
size = await storage.save_file("tracks/ab/track.mp3", src)
|
||||
|
||||
s3.put_object.assert_awaited_once_with(
|
||||
Bucket="test-bucket", Key="tracks/ab/track.mp3", Body=b"audio bytes"
|
||||
)
|
||||
assert size == 11
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# stat
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_stat_returns_size_and_content_type(storage: S3FileStorage) -> None:
|
||||
s3 = AsyncMock()
|
||||
s3.head_object.return_value = {"ContentLength": 1024, "ContentType": "audio/mpeg"}
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
stat = await storage.stat("tracks/ab/track.mp3")
|
||||
|
||||
assert stat.size == 1024
|
||||
assert stat.content_type == "audio/mpeg"
|
||||
|
||||
|
||||
async def test_stat_falls_back_to_ext_content_type(storage: S3FileStorage) -> None:
|
||||
s3 = AsyncMock()
|
||||
s3.head_object.return_value = {"ContentLength": 500, "ContentType": None}
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
stat = await storage.stat("tracks/ab/track.flac")
|
||||
|
||||
assert stat.content_type == "audio/flac"
|
||||
|
||||
|
||||
async def test_stat_not_found_raises_storage_error(storage: S3FileStorage) -> None:
|
||||
s3 = AsyncMock()
|
||||
s3.head_object.side_effect = _client_error("404")
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
with pytest.raises(StorageError, match="not found"):
|
||||
await storage.stat("tracks/missing.mp3")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# exists
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_exists_true(storage: S3FileStorage) -> None:
|
||||
s3 = AsyncMock()
|
||||
s3.head_object.return_value = {"ContentLength": 1}
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
assert await storage.exists("tracks/ab/track.mp3") is True
|
||||
|
||||
|
||||
async def test_exists_false_on_404(storage: S3FileStorage) -> None:
|
||||
s3 = AsyncMock()
|
||||
s3.head_object.side_effect = _client_error("NoSuchKey")
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
assert await storage.exists("tracks/missing.mp3") is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# delete
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_delete_calls_delete_object(storage: S3FileStorage) -> None:
|
||||
s3 = AsyncMock()
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
await storage.delete("tracks/ab/track.mp3")
|
||||
|
||||
s3.delete_object.assert_awaited_once_with(Bucket="test-bucket", Key="tracks/ab/track.mp3")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# open_range
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_open_range_full(storage: S3FileStorage) -> None:
|
||||
data = b"hello world"
|
||||
s3 = AsyncMock()
|
||||
s3.head_object.return_value = {"ContentLength": len(data)}
|
||||
s3.get_object.return_value = {"Body": _FakeBody(data)}
|
||||
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
stream, total = await storage.open_range("tracks/ab/t.mp3", 0, None)
|
||||
chunks = [c async for c in stream]
|
||||
|
||||
assert b"".join(chunks) == data
|
||||
assert total == len(data)
|
||||
s3.get_object.assert_awaited_once_with(
|
||||
Bucket="test-bucket", Key="tracks/ab/t.mp3", Range="bytes=0-"
|
||||
)
|
||||
|
||||
|
||||
async def test_open_range_partial(storage: S3FileStorage) -> None:
|
||||
full = b"0123456789"
|
||||
ranged = b"34567"
|
||||
s3 = AsyncMock()
|
||||
s3.head_object.return_value = {"ContentLength": len(full)}
|
||||
s3.get_object.return_value = {"Body": _FakeBody(ranged)}
|
||||
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
stream, total = await storage.open_range("tracks/ab/t.mp3", 3, 7)
|
||||
result = b"".join([c async for c in stream])
|
||||
|
||||
assert result == ranged
|
||||
assert total == len(full)
|
||||
s3.get_object.assert_awaited_once_with(
|
||||
Bucket="test-bucket", Key="tracks/ab/t.mp3", Range="bytes=3-7"
|
||||
)
|
||||
|
||||
|
||||
async def test_open_range_not_found_raises_storage_error(storage: S3FileStorage) -> None:
|
||||
s3 = AsyncMock()
|
||||
s3.head_object.side_effect = _client_error("NoSuchKey")
|
||||
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
with pytest.raises(StorageError, match="not found"):
|
||||
await storage.open_range("tracks/missing.mp3", 0, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# as_local_path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_as_local_path_yields_file_with_content(storage: S3FileStorage) -> None:
|
||||
data = b"local copy bytes"
|
||||
s3 = AsyncMock()
|
||||
s3.get_object.return_value = {"Body": _FakeBody(data)}
|
||||
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
async with storage.as_local_path("tracks/ab/track.mp3") as path:
|
||||
assert path.exists()
|
||||
assert path.read_bytes() == data
|
||||
|
||||
assert not path.exists()
|
||||
|
||||
|
||||
async def test_as_local_path_cleans_up_on_error(storage: S3FileStorage) -> None:
|
||||
s3 = AsyncMock()
|
||||
s3.get_object.side_effect = _client_error("NoSuchKey")
|
||||
|
||||
captured: list[Path] = []
|
||||
|
||||
with patch.object(storage, "_client", return_value=_make_client_ctx(s3)):
|
||||
with pytest.raises(StorageError):
|
||||
async with storage.as_local_path("tracks/missing.mp3") as path:
|
||||
captured.append(path)
|
||||
|
||||
if captured:
|
||||
assert not captured[0].exists()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# provider wiring
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_provider_returns_s3_storage_when_configured(tmp_path: Path) -> None:
|
||||
from app.core.config import Settings
|
||||
from app.infrastructure.storage import provider
|
||||
|
||||
provider._storage = None
|
||||
|
||||
mock_settings = Settings(
|
||||
database_url="postgresql+asyncpg://x:x@localhost/x",
|
||||
storage_backend="s3",
|
||||
s3_bucket="my-bucket",
|
||||
s3_region="us-east-1",
|
||||
)
|
||||
|
||||
with patch("app.infrastructure.storage.provider.get_settings", return_value=mock_settings):
|
||||
storage_instance = provider.get_file_storage()
|
||||
|
||||
assert isinstance(storage_instance, S3FileStorage)
|
||||
provider._storage = None # reset singleton for other tests
|
||||
|
||||
|
||||
def test_provider_raises_when_s3_bucket_missing() -> None:
|
||||
from app.core.config import Settings
|
||||
from app.infrastructure.storage import provider
|
||||
|
||||
provider._storage = None
|
||||
|
||||
mock_settings = Settings(
|
||||
database_url="postgresql+asyncpg://x:x@localhost/x",
|
||||
storage_backend="s3",
|
||||
s3_bucket=None,
|
||||
)
|
||||
|
||||
with patch("app.infrastructure.storage.provider.get_settings", return_value=mock_settings):
|
||||
with pytest.raises(RuntimeError, match="S3_BUCKET"):
|
||||
provider.get_file_storage()
|
||||
|
||||
provider._storage = None
|
||||
Reference in New Issue
Block a user