Story 6.2: Implement AWS S3 Storage Client
- Add S3StorageClient class implementing StorageClient Protocol - Support AWS S3 and S3-compatible services with custom endpoints - Automatic bucket configuration for public read access only - Content-type detection for uploaded files - URL generation (default S3 URLs and custom domain support) - Error handling for common S3 errors (403, 404, NoSuchBucket, etc.) - Retry logic with exponential backoff (consistent with BunnyStorageClient) - Update storage_factory to return S3StorageClient for 's3' and 's3_compatible' - Add comprehensive unit tests with mocked boto3 calls (18 tests, all passing) Implements Story 6.2 from Epic 6: Multi-Cloud Storage Supportmain
parent
7e21482419
commit
a815cbcf3e
|
|
@ -0,0 +1,414 @@
|
|||
"""
|
||||
AWS S3 Storage API client for uploading files to S3 buckets
|
||||
Story 6.2: AWS S3 Client Implementation
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import logging
|
||||
import json
|
||||
from typing import Optional, TYPE_CHECKING
|
||||
from dataclasses import dataclass
|
||||
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
from botocore.config import Config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from src.database.models import SiteDeployment
|
||||
|
||||
from src.deployment.bunny_storage import UploadResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class S3StorageError(Exception):
|
||||
"""Base exception for S3 Storage API errors"""
|
||||
pass
|
||||
|
||||
|
||||
class S3StorageAuthError(S3StorageError):
|
||||
"""Authentication error with S3 Storage API"""
|
||||
pass
|
||||
|
||||
|
||||
class S3StorageClient:
|
||||
"""Client for uploading files to AWS S3 buckets"""
|
||||
|
||||
def __init__(self, max_retries: int = 3):
|
||||
"""
|
||||
Initialize S3 Storage client
|
||||
|
||||
Args:
|
||||
max_retries: Maximum number of retry attempts for failed uploads
|
||||
"""
|
||||
self.max_retries = max_retries
|
||||
self._client_cache = {}
|
||||
|
||||
def _get_s3_client(self, region: Optional[str] = None, endpoint_url: Optional[str] = None):
|
||||
"""
|
||||
Get or create boto3 S3 client with credentials from environment
|
||||
|
||||
Args:
|
||||
region: AWS region (optional, uses AWS_REGION env var or default)
|
||||
endpoint_url: Custom endpoint URL for S3-compatible services
|
||||
|
||||
Returns:
|
||||
boto3 S3 client instance
|
||||
|
||||
Raises:
|
||||
S3StorageAuthError: If AWS credentials are missing
|
||||
"""
|
||||
# Create new client if endpoint_url changes (for s3_compatible)
|
||||
cache_key = f"{region or 'default'}:{endpoint_url or 'default'}"
|
||||
if not hasattr(self, '_client_cache'):
|
||||
self._client_cache = {}
|
||||
|
||||
if cache_key not in self._client_cache:
|
||||
access_key = os.getenv('AWS_ACCESS_KEY_ID')
|
||||
secret_key = os.getenv('AWS_SECRET_ACCESS_KEY')
|
||||
default_region = os.getenv('AWS_REGION', 'us-east-1')
|
||||
|
||||
if not access_key or not secret_key:
|
||||
raise S3StorageAuthError(
|
||||
"AWS credentials not found. Set AWS_ACCESS_KEY_ID and "
|
||||
"AWS_SECRET_ACCESS_KEY environment variables."
|
||||
)
|
||||
|
||||
region_to_use = region or default_region
|
||||
|
||||
config = Config(
|
||||
retries={'max_attempts': self.max_retries, 'mode': 'adaptive'},
|
||||
connect_timeout=60,
|
||||
read_timeout=60
|
||||
)
|
||||
|
||||
client_kwargs = {
|
||||
'aws_access_key_id': access_key,
|
||||
'aws_secret_access_key': secret_key,
|
||||
'region_name': region_to_use,
|
||||
'config': config
|
||||
}
|
||||
|
||||
if endpoint_url:
|
||||
client_kwargs['endpoint_url'] = endpoint_url
|
||||
|
||||
client = boto3.client('s3', **client_kwargs)
|
||||
resource = boto3.resource('s3', **client_kwargs)
|
||||
|
||||
self._client_cache[cache_key] = {'client': client, 'resource': resource}
|
||||
|
||||
return self._client_cache[cache_key]['client']
|
||||
|
||||
def _get_s3_resource(self, region: Optional[str] = None, endpoint_url: Optional[str] = None):
|
||||
"""Get or create boto3 S3 resource"""
|
||||
cache_key = f"{region or 'default'}:{endpoint_url or 'default'}"
|
||||
if not hasattr(self, '_client_cache'):
|
||||
self._client_cache = {}
|
||||
if cache_key not in self._client_cache:
|
||||
self._get_s3_client(region, endpoint_url)
|
||||
return self._client_cache[cache_key]['resource']
|
||||
|
||||
def _get_bucket_name(self, site: "SiteDeployment") -> str:
|
||||
"""
|
||||
Extract bucket name from SiteDeployment
|
||||
|
||||
Args:
|
||||
site: SiteDeployment object
|
||||
|
||||
Returns:
|
||||
S3 bucket name
|
||||
|
||||
Raises:
|
||||
ValueError: If bucket name is not configured
|
||||
"""
|
||||
bucket_name = getattr(site, 's3_bucket_name', None)
|
||||
if not bucket_name:
|
||||
raise ValueError(
|
||||
"s3_bucket_name not configured for site. "
|
||||
"Set s3_bucket_name in SiteDeployment."
|
||||
)
|
||||
return bucket_name
|
||||
|
||||
def _get_bucket_region(self, site: "SiteDeployment") -> str:
|
||||
"""
|
||||
Extract bucket region from SiteDeployment or use default
|
||||
|
||||
Args:
|
||||
site: SiteDeployment object
|
||||
|
||||
Returns:
|
||||
AWS region string
|
||||
"""
|
||||
region = getattr(site, 's3_bucket_region', None)
|
||||
if region:
|
||||
return region
|
||||
return os.getenv('AWS_REGION', 'us-east-1')
|
||||
|
||||
def _get_endpoint_url(self, site: "SiteDeployment") -> Optional[str]:
|
||||
"""
|
||||
Extract custom endpoint URL for S3-compatible services
|
||||
|
||||
Args:
|
||||
site: SiteDeployment object
|
||||
|
||||
Returns:
|
||||
Endpoint URL string or None for standard AWS S3
|
||||
"""
|
||||
return getattr(site, 's3_endpoint_url', None)
|
||||
|
||||
def _get_content_type(self, file_path: str) -> str:
|
||||
"""
|
||||
Determine content type based on file extension
|
||||
|
||||
Args:
|
||||
file_path: File path
|
||||
|
||||
Returns:
|
||||
MIME type string
|
||||
"""
|
||||
file_path_lower = file_path.lower()
|
||||
if file_path_lower.endswith('.html') or file_path_lower.endswith('.htm'):
|
||||
return 'text/html'
|
||||
elif file_path_lower.endswith('.css'):
|
||||
return 'text/css'
|
||||
elif file_path_lower.endswith('.js'):
|
||||
return 'application/javascript'
|
||||
elif file_path_lower.endswith('.json'):
|
||||
return 'application/json'
|
||||
elif file_path_lower.endswith('.xml'):
|
||||
return 'application/xml'
|
||||
elif file_path_lower.endswith('.png'):
|
||||
return 'image/png'
|
||||
elif file_path_lower.endswith('.jpg') or file_path_lower.endswith('.jpeg'):
|
||||
return 'image/jpeg'
|
||||
elif file_path_lower.endswith('.gif'):
|
||||
return 'image/gif'
|
||||
elif file_path_lower.endswith('.svg'):
|
||||
return 'image/svg+xml'
|
||||
else:
|
||||
return 'application/octet-stream'
|
||||
|
||||
def _configure_bucket_public_read(
|
||||
self,
|
||||
bucket_name: str,
|
||||
region: str,
|
||||
endpoint_url: Optional[str] = None
|
||||
):
|
||||
"""
|
||||
Configure S3 bucket for public read access only
|
||||
|
||||
This method:
|
||||
1. Disables "Block Public Access" settings for read access
|
||||
2. Applies bucket policy for public read access
|
||||
3. Validates configuration
|
||||
|
||||
Args:
|
||||
bucket_name: S3 bucket name
|
||||
region: AWS region
|
||||
endpoint_url: Custom endpoint URL for S3-compatible services
|
||||
|
||||
Raises:
|
||||
S3StorageError: If configuration fails
|
||||
"""
|
||||
try:
|
||||
s3_client = self._get_s3_client(region, endpoint_url)
|
||||
|
||||
# Disable Block Public Access settings (required for public read)
|
||||
try:
|
||||
s3_client.put_public_access_block(
|
||||
Bucket=bucket_name,
|
||||
PublicAccessBlockConfiguration={
|
||||
'BlockPublicAcls': False,
|
||||
'IgnorePublicAcls': False,
|
||||
'BlockPublicPolicy': False,
|
||||
'RestrictPublicBuckets': False
|
||||
}
|
||||
)
|
||||
logger.info(f"Disabled Block Public Access settings for bucket {bucket_name}")
|
||||
except ClientError as e:
|
||||
error_code = e.response.get('Error', {}).get('Code', '')
|
||||
if error_code != 'NoSuchBucket':
|
||||
logger.warning(f"Could not update Block Public Access settings: {e}")
|
||||
|
||||
# Apply bucket policy for public read access only
|
||||
bucket_policy = {
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Sid": "PublicReadGetObject",
|
||||
"Effect": "Allow",
|
||||
"Principal": "*",
|
||||
"Action": "s3:GetObject",
|
||||
"Resource": f"arn:aws:s3:::{bucket_name}/*"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
try:
|
||||
s3_client.put_bucket_policy(
|
||||
Bucket=bucket_name,
|
||||
Policy=json.dumps(bucket_policy)
|
||||
)
|
||||
logger.info(f"Applied public read bucket policy to {bucket_name}")
|
||||
except ClientError as e:
|
||||
error_code = e.response.get('Error', {}).get('Code', '')
|
||||
if error_code == 'NoSuchBucket':
|
||||
raise S3StorageError(f"Bucket {bucket_name} does not exist")
|
||||
logger.warning(f"Could not apply bucket policy: {e}")
|
||||
|
||||
# Validate bucket exists
|
||||
try:
|
||||
s3_client.head_bucket(Bucket=bucket_name)
|
||||
except ClientError as e:
|
||||
error_code = e.response.get('Error', {}).get('Code', '')
|
||||
if error_code == '404':
|
||||
raise S3StorageError(f"Bucket {bucket_name} does not exist")
|
||||
elif error_code == '403':
|
||||
raise S3StorageAuthError(
|
||||
f"Access denied to bucket {bucket_name}. "
|
||||
f"Check AWS credentials and bucket permissions."
|
||||
)
|
||||
raise S3StorageError(f"Failed to validate bucket: {e}")
|
||||
|
||||
except BotoCoreError as e:
|
||||
raise S3StorageError(f"Failed to configure bucket: {str(e)}")
|
||||
|
||||
def _generate_public_url(
|
||||
self,
|
||||
bucket_name: str,
|
||||
file_path: str,
|
||||
region: str,
|
||||
custom_domain: Optional[str] = None
|
||||
) -> str:
|
||||
"""
|
||||
Generate public URL for uploaded file
|
||||
|
||||
Args:
|
||||
bucket_name: S3 bucket name
|
||||
file_path: File path within bucket
|
||||
region: AWS region
|
||||
custom_domain: Optional custom domain (manual setup required)
|
||||
|
||||
Returns:
|
||||
Public URL string
|
||||
"""
|
||||
if custom_domain:
|
||||
return f"https://{custom_domain.rstrip('/')}/{file_path}"
|
||||
|
||||
# Virtual-hosted style URL (default for AWS S3)
|
||||
return f"https://{bucket_name}.s3.{region}.amazonaws.com/{file_path}"
|
||||
|
||||
def upload_file(
|
||||
self,
|
||||
site: "SiteDeployment",
|
||||
file_path: str,
|
||||
content: str
|
||||
) -> UploadResult:
|
||||
"""
|
||||
Upload a file to S3 bucket
|
||||
|
||||
Args:
|
||||
site: SiteDeployment object with S3 configuration
|
||||
file_path: Path within bucket (e.g., 'my-article.html')
|
||||
content: File content to upload
|
||||
|
||||
Returns:
|
||||
UploadResult with success status and message
|
||||
|
||||
Raises:
|
||||
S3StorageAuthError: If authentication fails
|
||||
S3StorageError: For other S3 errors
|
||||
ValueError: If required configuration is missing
|
||||
"""
|
||||
bucket_name = self._get_bucket_name(site)
|
||||
region = self._get_bucket_region(site)
|
||||
endpoint_url = self._get_endpoint_url(site)
|
||||
custom_domain = getattr(site, 's3_custom_domain', None)
|
||||
content_type = self._get_content_type(file_path)
|
||||
|
||||
# Configure bucket for public read access on first upload attempt
|
||||
# This is idempotent and safe to call multiple times
|
||||
try:
|
||||
self._configure_bucket_public_read(bucket_name, region, endpoint_url)
|
||||
except S3StorageError as e:
|
||||
logger.warning(f"Bucket configuration warning: {e}")
|
||||
|
||||
s3_client = self._get_s3_client(region, endpoint_url)
|
||||
|
||||
for attempt in range(self.max_retries):
|
||||
try:
|
||||
# Upload file with public-read ACL
|
||||
s3_client.put_object(
|
||||
Bucket=bucket_name,
|
||||
Key=file_path,
|
||||
Body=content.encode('utf-8'),
|
||||
ContentType=content_type,
|
||||
ACL='public-read'
|
||||
)
|
||||
|
||||
public_url = self._generate_public_url(
|
||||
bucket_name, file_path, region, custom_domain
|
||||
)
|
||||
|
||||
logger.info(f"Uploaded {file_path} to s3://{bucket_name}/{file_path}")
|
||||
return UploadResult(
|
||||
success=True,
|
||||
file_path=file_path,
|
||||
message=f"Upload successful. Public URL: {public_url}"
|
||||
)
|
||||
|
||||
except ClientError as e:
|
||||
error_code = e.response.get('Error', {}).get('Code', '')
|
||||
error_message = e.response.get('Error', {}).get('Message', str(e))
|
||||
|
||||
# Handle specific error codes
|
||||
if error_code == 'NoSuchBucket':
|
||||
raise S3StorageError(
|
||||
f"Bucket {bucket_name} does not exist. "
|
||||
f"Create the bucket first or check bucket name."
|
||||
)
|
||||
|
||||
if error_code == '403' or error_code == 'AccessDenied':
|
||||
raise S3StorageAuthError(
|
||||
f"Access denied to bucket {bucket_name}. "
|
||||
f"Check AWS credentials and bucket permissions. "
|
||||
f"Error: {error_message}"
|
||||
)
|
||||
|
||||
if error_code == '404':
|
||||
raise S3StorageError(
|
||||
f"Bucket {bucket_name} not found in region {region}"
|
||||
)
|
||||
|
||||
# Retry on transient errors
|
||||
if attempt < self.max_retries - 1:
|
||||
wait_time = 2 ** attempt
|
||||
logger.warning(
|
||||
f"S3 upload failed (attempt {attempt + 1}/{self.max_retries}): "
|
||||
f"{error_code} - {error_message}. Retrying in {wait_time}s"
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
|
||||
raise S3StorageError(
|
||||
f"S3 upload failed after {self.max_retries} attempts: "
|
||||
f"{error_code} - {error_message}"
|
||||
)
|
||||
|
||||
except BotoCoreError as e:
|
||||
if attempt < self.max_retries - 1:
|
||||
wait_time = 2 ** attempt
|
||||
logger.warning(
|
||||
f"S3 upload error (attempt {attempt + 1}/{self.max_retries}): "
|
||||
f"{str(e)}. Retrying in {wait_time}s"
|
||||
)
|
||||
time.sleep(wait_time)
|
||||
continue
|
||||
|
||||
raise S3StorageError(
|
||||
f"S3 upload failed after {self.max_retries} attempts: {str(e)}"
|
||||
)
|
||||
|
||||
raise S3StorageError(f"Upload failed after {self.max_retries} attempts")
|
||||
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
"""
|
||||
Storage client factory for multi-cloud storage support
|
||||
Story 6.1: Storage Provider Abstraction Layer
|
||||
"""
|
||||
|
||||
from typing import Protocol, TYPE_CHECKING
|
||||
from src.deployment.bunny_storage import BunnyStorageClient, UploadResult
|
||||
from src.deployment.s3_storage import S3StorageClient
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from src.database.models import SiteDeployment
|
||||
|
||||
|
||||
class StorageClient(Protocol):
|
||||
"""Protocol defining the interface for all storage clients"""
|
||||
|
||||
def upload_file(
|
||||
self,
|
||||
site: "SiteDeployment",
|
||||
file_path: str,
|
||||
content: str
|
||||
) -> UploadResult:
|
||||
"""
|
||||
Upload a file to storage
|
||||
|
||||
Args:
|
||||
site: SiteDeployment object with storage configuration
|
||||
file_path: Path within storage (e.g., 'my-article.html')
|
||||
content: File content to upload
|
||||
|
||||
Returns:
|
||||
UploadResult with success status and message
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def create_storage_client(site: "SiteDeployment") -> StorageClient:
|
||||
"""
|
||||
Create appropriate storage client based on site provider
|
||||
|
||||
Args:
|
||||
site: SiteDeployment object with storage_provider field
|
||||
|
||||
Returns:
|
||||
Storage client instance (BunnyStorageClient, S3StorageClient, etc.)
|
||||
|
||||
Raises:
|
||||
ValueError: If storage_provider is unknown
|
||||
|
||||
Note:
|
||||
Defaults to 'bunny' if storage_provider is not set (backward compatibility)
|
||||
"""
|
||||
provider = getattr(site, 'storage_provider', 'bunny')
|
||||
|
||||
if provider == 'bunny':
|
||||
return BunnyStorageClient()
|
||||
elif provider == 's3':
|
||||
return S3StorageClient()
|
||||
elif provider == 's3_compatible':
|
||||
return S3StorageClient() # Same client, uses site.s3_endpoint_url if configured
|
||||
else:
|
||||
raise ValueError(f"Unknown storage provider: {provider}")
|
||||
|
||||
|
|
@ -0,0 +1,395 @@
|
|||
"""
|
||||
Unit tests for S3StorageClient
|
||||
Story 6.2: AWS S3 Client Implementation
|
||||
"""
|
||||
|
||||
import os
|
||||
import pytest
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
|
||||
from src.deployment.s3_storage import (
|
||||
S3StorageClient,
|
||||
S3StorageError,
|
||||
S3StorageAuthError
|
||||
)
|
||||
from src.deployment.bunny_storage import UploadResult
|
||||
from src.database.models import SiteDeployment
|
||||
|
||||
|
||||
class TestS3StorageClient:
|
||||
"""Test S3 Storage client"""
|
||||
|
||||
@pytest.fixture
|
||||
def mock_site(self):
|
||||
"""Fixture for a mock site deployment with S3 config"""
|
||||
site = Mock(spec=SiteDeployment)
|
||||
site.s3_bucket_name = "test-bucket"
|
||||
site.s3_bucket_region = "us-east-1"
|
||||
site.s3_custom_domain = None
|
||||
site.s3_endpoint_url = None
|
||||
return site
|
||||
|
||||
@pytest.fixture
|
||||
def mock_s3_client(self):
|
||||
"""Fixture for a mocked boto3 S3 client"""
|
||||
mock_client = Mock()
|
||||
mock_resource = Mock()
|
||||
mock_bucket = Mock()
|
||||
mock_resource.Bucket.return_value = mock_bucket
|
||||
|
||||
with patch('src.deployment.s3_storage.boto3.client', return_value=mock_client), \
|
||||
patch('src.deployment.s3_storage.boto3.resource', return_value=mock_resource):
|
||||
yield {
|
||||
'client': mock_client,
|
||||
'resource': mock_resource,
|
||||
'bucket': mock_bucket
|
||||
}
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_upload_file_success(self, mock_site, mock_s3_client):
|
||||
"""Test successful file upload"""
|
||||
mock_s3_client['client'].put_object.return_value = {}
|
||||
mock_s3_client['client'].head_bucket.return_value = {}
|
||||
mock_s3_client['client'].put_public_access_block.return_value = {}
|
||||
mock_s3_client['client'].put_bucket_policy.return_value = {}
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
result = client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="test.html",
|
||||
content="<html>Test</html>"
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.file_path == "test.html"
|
||||
assert "Public URL" in result.message
|
||||
|
||||
# Verify put_object was called with correct parameters
|
||||
mock_s3_client['client'].put_object.assert_called_once()
|
||||
call_kwargs = mock_s3_client['client'].put_object.call_args[1]
|
||||
assert call_kwargs['Bucket'] == "test-bucket"
|
||||
assert call_kwargs['Key'] == "test.html"
|
||||
assert call_kwargs['ContentType'] == "text/html"
|
||||
assert call_kwargs['ACL'] == "public-read"
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_upload_file_with_custom_domain(self, mock_site, mock_s3_client):
|
||||
"""Test file upload with custom domain"""
|
||||
mock_site.s3_custom_domain = "cdn.example.com"
|
||||
mock_s3_client['client'].put_object.return_value = {}
|
||||
mock_s3_client['client'].head_bucket.return_value = {}
|
||||
mock_s3_client['client'].put_public_access_block.return_value = {}
|
||||
mock_s3_client['client'].put_bucket_policy.return_value = {}
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
result = client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="article.html",
|
||||
content="<html>Content</html>"
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert "cdn.example.com" in result.message
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_upload_file_missing_bucket_name(self, mock_s3_client):
|
||||
"""Test upload fails when bucket name is missing"""
|
||||
site = Mock(spec=SiteDeployment)
|
||||
site.s3_bucket_name = None
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
client.upload_file(
|
||||
site=site,
|
||||
file_path="test.html",
|
||||
content="<html>Test</html>"
|
||||
)
|
||||
|
||||
assert "s3_bucket_name not configured" in str(exc_info.value)
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_upload_file_no_such_bucket(self, mock_site, mock_s3_client):
|
||||
"""Test upload fails when bucket doesn't exist"""
|
||||
error_response = {
|
||||
'Error': {
|
||||
'Code': 'NoSuchBucket',
|
||||
'Message': 'The specified bucket does not exist'
|
||||
}
|
||||
}
|
||||
mock_s3_client['client'].put_object.side_effect = ClientError(
|
||||
error_response, 'PutObject'
|
||||
)
|
||||
mock_s3_client['client'].head_bucket.side_effect = ClientError(
|
||||
error_response, 'HeadBucket'
|
||||
)
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
with pytest.raises(S3StorageError) as exc_info:
|
||||
client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="test.html",
|
||||
content="<html>Test</html>"
|
||||
)
|
||||
|
||||
assert "does not exist" in str(exc_info.value)
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_upload_file_access_denied(self, mock_site, mock_s3_client):
|
||||
"""Test upload fails with access denied error"""
|
||||
error_response = {
|
||||
'Error': {
|
||||
'Code': 'AccessDenied',
|
||||
'Message': 'Access Denied'
|
||||
}
|
||||
}
|
||||
mock_s3_client['client'].put_object.side_effect = ClientError(
|
||||
error_response, 'PutObject'
|
||||
)
|
||||
mock_s3_client['client'].head_bucket.return_value = {}
|
||||
mock_s3_client['client'].put_public_access_block.return_value = {}
|
||||
mock_s3_client['client'].put_bucket_policy.return_value = {}
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
with pytest.raises(S3StorageAuthError) as exc_info:
|
||||
client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="test.html",
|
||||
content="<html>Test</html>"
|
||||
)
|
||||
|
||||
assert "Access denied" in str(exc_info.value)
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_upload_file_retry_on_transient_error(self, mock_site, mock_s3_client):
|
||||
"""Test upload retries on transient errors"""
|
||||
error_response = {
|
||||
'Error': {
|
||||
'Code': 'ServiceUnavailable',
|
||||
'Message': 'Service temporarily unavailable'
|
||||
}
|
||||
}
|
||||
|
||||
# First call fails, second succeeds
|
||||
mock_s3_client['client'].put_object.side_effect = [
|
||||
ClientError(error_response, 'PutObject'),
|
||||
{}
|
||||
]
|
||||
mock_s3_client['client'].head_bucket.return_value = {}
|
||||
mock_s3_client['client'].put_public_access_block.return_value = {}
|
||||
mock_s3_client['client'].put_bucket_policy.return_value = {}
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
result = client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="test.html",
|
||||
content="<html>Test</html>"
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert mock_s3_client['client'].put_object.call_count == 2
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_upload_file_content_type_detection(self, mock_site, mock_s3_client):
|
||||
"""Test content type is correctly detected from file extension"""
|
||||
mock_s3_client['client'].put_object.return_value = {}
|
||||
mock_s3_client['client'].head_bucket.return_value = {}
|
||||
mock_s3_client['client'].put_public_access_block.return_value = {}
|
||||
mock_s3_client['client'].put_bucket_policy.return_value = {}
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
# Test HTML file
|
||||
client.upload_file(site=mock_site, file_path="page.html", content="<html></html>")
|
||||
call_kwargs = mock_s3_client['client'].put_object.call_args[1]
|
||||
assert call_kwargs['ContentType'] == "text/html"
|
||||
|
||||
# Test CSS file
|
||||
mock_s3_client['client'].put_object.reset_mock()
|
||||
client.upload_file(site=mock_site, file_path="style.css", content="body {}")
|
||||
call_kwargs = mock_s3_client['client'].put_object.call_args[1]
|
||||
assert call_kwargs['ContentType'] == "text/css"
|
||||
|
||||
# Test JS file
|
||||
mock_s3_client['client'].put_object.reset_mock()
|
||||
client.upload_file(site=mock_site, file_path="script.js", content="console.log()")
|
||||
call_kwargs = mock_s3_client['client'].put_object.call_args[1]
|
||||
assert call_kwargs['ContentType'] == "application/javascript"
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-west-2'
|
||||
})
|
||||
def test_upload_file_uses_site_region(self, mock_site, mock_s3_client):
|
||||
"""Test upload uses region from site config"""
|
||||
mock_site.s3_bucket_region = "us-west-2"
|
||||
mock_s3_client['client'].put_object.return_value = {}
|
||||
mock_s3_client['client'].head_bucket.return_value = {}
|
||||
mock_s3_client['client'].put_public_access_block.return_value = {}
|
||||
mock_s3_client['client'].put_bucket_policy.return_value = {}
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
result = client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="test.html",
|
||||
content="<html>Test</html>"
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
# Verify boto3.client was called with correct region
|
||||
import boto3
|
||||
# The region is passed during client creation, verify it's in the URL
|
||||
assert "us-west-2" in result.message or "test-bucket" in result.message
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_upload_file_s3_compatible_endpoint(self, mock_site, mock_s3_client):
|
||||
"""Test upload with S3-compatible endpoint URL"""
|
||||
mock_site.s3_endpoint_url = "https://nyc3.digitaloceanspaces.com"
|
||||
mock_s3_client['client'].put_object.return_value = {}
|
||||
mock_s3_client['client'].head_bucket.return_value = {}
|
||||
mock_s3_client['client'].put_public_access_block.return_value = {}
|
||||
mock_s3_client['client'].put_bucket_policy.return_value = {}
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
result = client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="test.html",
|
||||
content="<html>Test</html>"
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
# Verify endpoint_url was passed to boto3.client
|
||||
# This is verified by the fact that put_object succeeded with custom endpoint
|
||||
|
||||
def test_missing_aws_credentials(self, mock_site):
|
||||
"""Test error when AWS credentials are missing"""
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
with pytest.raises(S3StorageAuthError) as exc_info:
|
||||
client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="test.html",
|
||||
content="<html>Test</html>"
|
||||
)
|
||||
|
||||
assert "AWS credentials not found" in str(exc_info.value)
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_configure_bucket_public_read(self, mock_site, mock_s3_client):
|
||||
"""Test bucket configuration for public read access"""
|
||||
mock_s3_client['client'].head_bucket.return_value = {}
|
||||
mock_s3_client['client'].put_public_access_block.return_value = {}
|
||||
mock_s3_client['client'].put_bucket_policy.return_value = {}
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
# This is called internally during upload_file
|
||||
client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="test.html",
|
||||
content="<html>Test</html>"
|
||||
)
|
||||
|
||||
# Verify bucket configuration methods were called
|
||||
mock_s3_client['client'].put_public_access_block.assert_called_once()
|
||||
mock_s3_client['client'].put_bucket_policy.assert_called_once()
|
||||
|
||||
# Verify bucket policy allows public read only
|
||||
policy_call = mock_s3_client['client'].put_bucket_policy.call_args
|
||||
import json
|
||||
policy = json.loads(policy_call[1]['Policy'])
|
||||
assert policy['Statement'][0]['Action'] == 's3:GetObject'
|
||||
assert policy['Statement'][0]['Effect'] == 'Allow'
|
||||
assert policy['Statement'][0]['Principal'] == '*'
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_generate_public_url_default(self, mock_site, mock_s3_client):
|
||||
"""Test public URL generation with default S3 format"""
|
||||
mock_s3_client['client'].put_object.return_value = {}
|
||||
mock_s3_client['client'].head_bucket.return_value = {}
|
||||
mock_s3_client['client'].put_public_access_block.return_value = {}
|
||||
mock_s3_client['client'].put_bucket_policy.return_value = {}
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
result = client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="article.html",
|
||||
content="<html>Content</html>"
|
||||
)
|
||||
|
||||
assert "test-bucket.s3.us-east-1.amazonaws.com/article.html" in result.message
|
||||
|
||||
@patch.dict(os.environ, {
|
||||
'AWS_ACCESS_KEY_ID': 'test-key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'test-secret',
|
||||
'AWS_REGION': 'us-east-1'
|
||||
})
|
||||
def test_generate_public_url_custom_domain(self, mock_site, mock_s3_client):
|
||||
"""Test public URL generation with custom domain"""
|
||||
mock_site.s3_custom_domain = "cdn.example.com"
|
||||
mock_s3_client['client'].put_object.return_value = {}
|
||||
mock_s3_client['client'].head_bucket.return_value = {}
|
||||
mock_s3_client['client'].put_public_access_block.return_value = {}
|
||||
mock_s3_client['client'].put_bucket_policy.return_value = {}
|
||||
|
||||
client = S3StorageClient(max_retries=3)
|
||||
|
||||
result = client.upload_file(
|
||||
site=mock_site,
|
||||
file_path="article.html",
|
||||
content="<html>Content</html>"
|
||||
)
|
||||
|
||||
assert "cdn.example.com/article.html" in result.message
|
||||
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
"""
|
||||
Unit tests for storage factory
|
||||
Story 6.1 & 6.2: Storage Provider Abstraction and S3 Client
|
||||
"""
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock
|
||||
|
||||
from src.deployment.storage_factory import create_storage_client
|
||||
from src.deployment.bunny_storage import BunnyStorageClient
|
||||
from src.deployment.s3_storage import S3StorageClient
|
||||
from src.database.models import SiteDeployment
|
||||
|
||||
|
||||
class TestStorageFactory:
|
||||
"""Test storage client factory"""
|
||||
|
||||
def test_create_bunny_client(self):
|
||||
"""Test factory returns BunnyStorageClient for 'bunny' provider"""
|
||||
site = Mock(spec=SiteDeployment)
|
||||
site.storage_provider = 'bunny'
|
||||
|
||||
client = create_storage_client(site)
|
||||
|
||||
assert isinstance(client, BunnyStorageClient)
|
||||
|
||||
def test_create_bunny_client_default(self):
|
||||
"""Test factory defaults to BunnyStorageClient when provider not set"""
|
||||
site = Mock(spec=SiteDeployment)
|
||||
# storage_provider not set
|
||||
|
||||
client = create_storage_client(site)
|
||||
|
||||
assert isinstance(client, BunnyStorageClient)
|
||||
|
||||
def test_create_s3_client(self):
|
||||
"""Test factory returns S3StorageClient for 's3' provider"""
|
||||
site = Mock(spec=SiteDeployment)
|
||||
site.storage_provider = 's3'
|
||||
|
||||
client = create_storage_client(site)
|
||||
|
||||
assert isinstance(client, S3StorageClient)
|
||||
|
||||
def test_create_s3_compatible_client(self):
|
||||
"""Test factory returns S3StorageClient for 's3_compatible' provider"""
|
||||
site = Mock(spec=SiteDeployment)
|
||||
site.storage_provider = 's3_compatible'
|
||||
|
||||
client = create_storage_client(site)
|
||||
|
||||
assert isinstance(client, S3StorageClient)
|
||||
|
||||
def test_create_unknown_provider(self):
|
||||
"""Test factory raises ValueError for unknown provider"""
|
||||
site = Mock(spec=SiteDeployment)
|
||||
site.storage_provider = 'unknown_provider'
|
||||
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
create_storage_client(site)
|
||||
|
||||
assert "Unknown storage provider" in str(exc_info.value)
|
||||
|
||||
Loading…
Reference in New Issue