""" CLI command definitions using Click """ import click from typing import Optional from src.core.config import get_config, get_bunny_account_api_key, get_concurrent_workers from src.auth.service import AuthService from src.database.session import db_manager from src.database.repositories import UserRepository, SiteDeploymentRepository, ProjectRepository from src.database.models import User from src.deployment.bunnynet import ( BunnyNetClient, BunnyNetAPIError, BunnyNetAuthError, BunnyNetResourceConflictError ) from src.ingestion.parser import CORAParser, CORAParseError, SimpleSpreadsheetParser from src.generation.ai_client import AIClient, PromptManager from src.generation.service import ContentGenerator from src.generation.batch_processor import BatchProcessor from src.database.repositories import GeneratedContentRepository, SitePageRepository from src.deployment.bunny_storage import BunnyStorageClient, BunnyStorageError from src.deployment.deployment_service import DeploymentService from src.deployment.url_logger import URLLogger from dotenv import load_dotenv import os import requests import random # Load .env file at module level load_dotenv() def authenticate_admin(username: str, password: str) -> Optional[User]: """ Authenticate a user and verify they have admin role Args: username: The username to authenticate password: The password to authenticate Returns: User object if authenticated and is admin, None otherwise """ session = db_manager.get_session() try: user_repo = UserRepository(session) auth_service = AuthService(user_repo) user = auth_service.authenticate_user(username, password) if user and user.is_admin(): return user return None finally: session.close() def prompt_admin_credentials() -> tuple[str, str]: """ Prompt for admin username and password Checks environment variables CLIENT_USERNAME and CLIENT_PASSWORD first Returns: Tuple of (username, password) """ username = os.getenv("CLIENT_USERNAME") password = os.getenv("CLIENT_PASSWORD") if username and password: return username, password click.echo("Admin authentication required") if not username: username = click.prompt("Username", type=str) if not password: password = click.prompt("Password", type=str, hide_input=True) return username, password @click.group() @click.version_option(version="1.0.0") def app(): """Content Automation & Syndication Platform CLI""" pass @app.command() def config(): """Show current configuration""" try: config = get_config() click.echo("Current Configuration:") click.echo(f"Application: {config.application.name} v{config.application.version}") click.echo(f"Environment: {config.application.environment}") click.echo(f"Database: {config.database.url}") click.echo(f"AI Model: {config.ai_service.model}") click.echo(f"Log Level: {config.logging.level}") except Exception as e: click.echo(f"Error loading configuration: {e}", err=True) @app.command() def health(): """Check system health""" try: config = get_config() click.echo("[OK] Configuration loaded successfully") click.echo("[OK] System is healthy") except Exception as e: click.echo(f"[ERROR] System health check failed: {e}", err=True) raise click.Abort() @app.command() def models(): """List available AI models""" try: config = get_config() click.echo("Available AI Models:") click.echo(f"Current: {config.ai_service.model}") click.echo(f"Provider: {config.ai_service.provider}") click.echo(f"Base URL: {config.ai_service.base_url}") click.echo("\nAvailable models:") for model_name, model_id in config.ai_service.available_models.items(): status = " (current)" if model_id == config.ai_service.model else "" click.echo(f" {model_name}: {model_id}{status}") except Exception as e: click.echo(f"Error listing models: {e}", err=True) @app.command("add-user") @click.option("--username", prompt=True, help="Username for the new user") @click.option("--password", prompt=True, hide_input=True, confirmation_prompt=True, help="Password for the new user") @click.option("--role", type=click.Choice(["Admin", "User"], case_sensitive=True), prompt=True, help="Role for the new user") @click.option("--admin-user", help="Admin username for authentication") @click.option("--admin-password", help="Admin password for authentication") def add_user(username: str, password: str, role: str, admin_user: Optional[str], admin_password: Optional[str]): """Create a new user (requires admin authentication)""" try: # Authenticate admin if not admin_user or not admin_password: admin_user, admin_password = prompt_admin_credentials() admin = authenticate_admin(admin_user, admin_password) if not admin: click.echo("Error: Authentication failed or insufficient permissions", err=True) raise click.Abort() # Create the new user session = db_manager.get_session() try: user_repo = UserRepository(session) auth_service = AuthService(user_repo) new_user = auth_service.create_user_with_hashed_password( username=username, password=password, role=role ) click.echo(f"Success: User '{new_user.username}' created with role '{new_user.role}'") finally: session.close() except ValueError as e: click.echo(f"Error: {e}", err=True) raise click.Abort() except Exception as e: click.echo(f"Error creating user: {e}", err=True) raise click.Abort() @app.command("delete-user") @click.option("--username", prompt=True, help="Username to delete") @click.option("--admin-user", help="Admin username for authentication") @click.option("--admin-password", help="Admin password for authentication") @click.confirmation_option(prompt="Are you sure you want to delete this user?") def delete_user(username: str, admin_user: Optional[str], admin_password: Optional[str]): """Delete a user by username (requires admin authentication)""" try: # Authenticate admin if not admin_user or not admin_password: admin_user, admin_password = prompt_admin_credentials() admin = authenticate_admin(admin_user, admin_password) if not admin: click.echo("Error: Authentication failed or insufficient permissions", err=True) raise click.Abort() # Prevent admin from deleting themselves if admin.username == username: click.echo("Error: Cannot delete your own account", err=True) raise click.Abort() # Delete the user session = db_manager.get_session() try: user_repo = UserRepository(session) # Check if user exists user_to_delete = user_repo.get_by_username(username) if not user_to_delete: click.echo(f"Error: User '{username}' not found", err=True) raise click.Abort() # Delete the user success = user_repo.delete(user_to_delete.id) if success: click.echo(f"Success: User '{username}' has been deleted") else: click.echo(f"Error: Failed to delete user '{username}'", err=True) raise click.Abort() finally: session.close() except Exception as e: click.echo(f"Error deleting user: {e}", err=True) raise click.Abort() @app.command("list-users") @click.option("--admin-user", help="Admin username for authentication") @click.option("--admin-password", help="Admin password for authentication") def list_users(admin_user: Optional[str], admin_password: Optional[str]): """List all users (requires admin authentication)""" try: # Authenticate admin if not admin_user or not admin_password: admin_user, admin_password = prompt_admin_credentials() admin = authenticate_admin(admin_user, admin_password) if not admin: click.echo("Error: Authentication failed or insufficient permissions", err=True) raise click.Abort() # List all users session = db_manager.get_session() try: user_repo = UserRepository(session) users = user_repo.get_all() if not users: click.echo("No users found") return click.echo(f"\nTotal users: {len(users)}") click.echo("-" * 60) click.echo(f"{'ID':<5} {'Username':<20} {'Role':<10} {'Created'}") click.echo("-" * 60) for user in users: created = user.created_at.strftime("%Y-%m-%d %H:%M:%S") click.echo(f"{user.id:<5} {user.username:<20} {user.role:<10} {created}") click.echo("-" * 60) finally: session.close() except Exception as e: click.echo(f"Error listing users: {e}", err=True) raise click.Abort() @app.command("provision-site") @click.option("--name", prompt=True, help="Site name") @click.option("--domain", prompt=True, help="Custom domain (FQDN, e.g., www.example.com)") @click.option("--storage-name", prompt=True, help="Storage Zone name (must be globally unique)") @click.option("--region", prompt=True, type=click.Choice(["DE", "NY", "LA", "SG", "SYD"]), help="Storage region") @click.option("--admin-user", help="Admin username for authentication") @click.option("--admin-password", help="Admin password for authentication") def provision_site(name: str, domain: str, storage_name: str, region: str, admin_user: Optional[str], admin_password: Optional[str]): """Provision a new site with Storage Zone and Pull Zone (requires admin)""" try: # Authenticate admin if not admin_user or not admin_password: admin_user, admin_password = prompt_admin_credentials() admin = authenticate_admin(admin_user, admin_password) if not admin: click.echo("Error: Authentication failed or insufficient permissions", err=True) raise click.Abort() # Get bunny.net API key try: api_key = get_bunny_account_api_key() except ValueError as e: click.echo(f"Error: {e}", err=True) click.echo("Please set BUNNY_ACCOUNT_API_KEY in your .env file", err=True) raise click.Abort() click.echo(f"\nProvisioning site '{name}' with domain '{domain}'...") # Initialize bunny.net client client = BunnyNetClient(api_key) session = db_manager.get_session() try: deployment_repo = SiteDeploymentRepository(session) # Check if domain already exists if deployment_repo.exists(domain): click.echo(f"Error: Site with domain '{domain}' already exists", err=True) raise click.Abort() # Step 1: Create Storage Zone click.echo(f"Step 1/3: Creating Storage Zone '{storage_name}' in region {region}...") storage_result = client.create_storage_zone(storage_name, region) click.echo(f" Storage Zone created: ID={storage_result.id}") # Step 2: Create Pull Zone pull_zone_name = f"{storage_name}-cdn" click.echo(f"Step 2/3: Creating Pull Zone '{pull_zone_name}'...") pull_result = client.create_pull_zone(pull_zone_name, storage_result.id) click.echo(f" Pull Zone created: ID={pull_result.id}, Hostname={pull_result.hostname}") # Step 3: Add Custom Hostname click.echo(f"Step 3/3: Adding custom hostname '{domain}'...") client.add_custom_hostname(pull_result.id, domain) click.echo(f" Custom hostname added successfully") # Save to database deployment = deployment_repo.create( site_name=name, custom_hostname=domain, storage_zone_id=storage_result.id, storage_zone_name=storage_result.name, storage_zone_password=storage_result.password, storage_zone_region=storage_result.region, pull_zone_id=pull_result.id, pull_zone_bcdn_hostname=pull_result.hostname ) click.echo("\n" + "=" * 70) click.echo("Site provisioned successfully!") click.echo("=" * 70) click.echo("\nMANUAL DNS CONFIGURATION REQUIRED:") click.echo("You must create the following CNAME record with your domain registrar:\n") click.echo(f" Type: CNAME") subdomain = domain.split('.')[0] if '.' in domain else '@' click.echo(f" Host: {subdomain}") click.echo(f" Value: {pull_result.hostname}") click.echo("\nExample DNS configuration:") click.echo(f" Type: CNAME") click.echo(f" Host: {subdomain}") click.echo(f" Value: {pull_result.hostname}") click.echo("\nNote: DNS propagation may take up to 48 hours.") click.echo("=" * 70) except BunnyNetAuthError as e: click.echo(f"Error: Authentication failed - {e}", err=True) click.echo("Please check your BUNNY_ACCOUNT_API_KEY", err=True) raise click.Abort() except BunnyNetResourceConflictError as e: click.echo(f"Error: Resource conflict - {e}", err=True) click.echo("Storage Zone or Pull Zone name already exists. Try a different name.", err=True) raise click.Abort() except BunnyNetAPIError as e: click.echo(f"Error: bunny.net API error - {e}", err=True) raise click.Abort() finally: session.close() except Exception as e: click.echo(f"Error provisioning site: {e}", err=True) raise click.Abort() @app.command("attach-domain") @click.option("--name", prompt=True, help="Site name") @click.option("--domain", prompt=True, help="Custom domain (FQDN, e.g., www.example.com)") @click.option("--storage-name", prompt=True, help="Existing Storage Zone name") @click.option("--admin-user", help="Admin username for authentication") @click.option("--admin-password", help="Admin password for authentication") def attach_domain(name: str, domain: str, storage_name: str, admin_user: Optional[str], admin_password: Optional[str]): """Attach a domain to an existing Storage Zone (requires admin)""" try: # Authenticate admin if not admin_user or not admin_password: admin_user, admin_password = prompt_admin_credentials() admin = authenticate_admin(admin_user, admin_password) if not admin: click.echo("Error: Authentication failed or insufficient permissions", err=True) raise click.Abort() # Get bunny.net API key try: api_key = get_bunny_account_api_key() except ValueError as e: click.echo(f"Error: {e}", err=True) click.echo("Please set BUNNY_ACCOUNT_API_KEY in your .env file", err=True) raise click.Abort() click.echo(f"\nAttaching domain '{domain}' to existing Storage Zone '{storage_name}'...") # Initialize bunny.net client client = BunnyNetClient(api_key) session = db_manager.get_session() try: deployment_repo = SiteDeploymentRepository(session) # Check if domain already exists if deployment_repo.exists(domain): click.echo(f"Error: Site with domain '{domain}' already exists", err=True) raise click.Abort() # Step 1: Find existing Storage Zone click.echo(f"Step 1/3: Finding Storage Zone '{storage_name}'...") storage_result = client.find_storage_zone_by_name(storage_name) if not storage_result: click.echo(f"Error: Storage Zone '{storage_name}' not found", err=True) raise click.Abort() click.echo(f" Storage Zone found: ID={storage_result.id}") # Step 2: Create Pull Zone pull_zone_name = f"{storage_name}-{domain.replace('.', '-')}" click.echo(f"Step 2/3: Creating Pull Zone '{pull_zone_name}'...") pull_result = client.create_pull_zone(pull_zone_name, storage_result.id) click.echo(f" Pull Zone created: ID={pull_result.id}, Hostname={pull_result.hostname}") # Step 3: Add Custom Hostname click.echo(f"Step 3/3: Adding custom hostname '{domain}'...") client.add_custom_hostname(pull_result.id, domain) click.echo(f" Custom hostname added successfully") # Save to database deployment = deployment_repo.create( site_name=name, custom_hostname=domain, storage_zone_id=storage_result.id, storage_zone_name=storage_result.name, storage_zone_password=storage_result.password, storage_zone_region=storage_result.region, pull_zone_id=pull_result.id, pull_zone_bcdn_hostname=pull_result.hostname ) click.echo("\n" + "=" * 70) click.echo("Domain attached successfully!") click.echo("=" * 70) click.echo("\nMANUAL DNS CONFIGURATION REQUIRED:") click.echo("You must create the following CNAME record with your domain registrar:\n") click.echo(f" Type: CNAME") subdomain = domain.split('.')[0] if '.' in domain else '@' click.echo(f" Host: {subdomain}") click.echo(f" Value: {pull_result.hostname}") click.echo("\nExample DNS configuration:") click.echo(f" Type: CNAME") click.echo(f" Host: {subdomain}") click.echo(f" Value: {pull_result.hostname}") click.echo("\nNote: DNS propagation may take up to 48 hours.") click.echo("=" * 70) except BunnyNetAuthError as e: click.echo(f"Error: Authentication failed - {e}", err=True) click.echo("Please check your BUNNY_ACCOUNT_API_KEY", err=True) raise click.Abort() except BunnyNetResourceConflictError as e: click.echo(f"Error: Resource conflict - {e}", err=True) click.echo("Pull Zone name already exists. Try a different domain.", err=True) raise click.Abort() except BunnyNetAPIError as e: click.echo(f"Error: bunny.net API error - {e}", err=True) raise click.Abort() finally: session.close() except Exception as e: click.echo(f"Error attaching domain: {e}", err=True) raise click.Abort() @app.command("list-sites") @click.option("--admin-user", help="Admin username for authentication") @click.option("--admin-password", help="Admin password for authentication") def list_sites(admin_user: Optional[str], admin_password: Optional[str]): """List all site deployments (requires admin)""" try: # Authenticate admin if not admin_user or not admin_password: admin_user, admin_password = prompt_admin_credentials() admin = authenticate_admin(admin_user, admin_password) if not admin: click.echo("Error: Authentication failed or insufficient permissions", err=True) raise click.Abort() # List all sites session = db_manager.get_session() try: deployment_repo = SiteDeploymentRepository(session) sites = deployment_repo.get_all() if not sites: click.echo("No site deployments found") return click.echo(f"\nTotal sites: {len(sites)}") click.echo("-" * 100) click.echo(f"{'ID':<5} {'Site Name':<25} {'Custom Domain':<30} {'Storage Zone':<20} {'Region':<8}") click.echo("-" * 100) for site in sites: click.echo(f"{site.id:<5} {site.site_name:<25} {site.custom_hostname or 'N/A':<30} " f"{site.storage_zone_name:<20} {site.storage_zone_region:<8}") click.echo("-" * 100) finally: session.close() except Exception as e: click.echo(f"Error listing sites: {e}", err=True) raise click.Abort() @app.command("get-site") @click.option("--domain", prompt=True, help="Custom domain to lookup") @click.option("--admin-user", help="Admin username for authentication") @click.option("--admin-password", help="Admin password for authentication") def get_site(domain: str, admin_user: Optional[str], admin_password: Optional[str]): """Get detailed information about a site deployment (requires admin)""" try: # Authenticate admin if not admin_user or not admin_password: admin_user, admin_password = prompt_admin_credentials() admin = authenticate_admin(admin_user, admin_password) if not admin: click.echo("Error: Authentication failed or insufficient permissions", err=True) raise click.Abort() # Get site details session = db_manager.get_session() try: deployment_repo = SiteDeploymentRepository(session) site = deployment_repo.get_by_hostname(domain) if not site: click.echo(f"Error: Site with domain '{domain}' not found", err=True) raise click.Abort() click.echo("\n" + "=" * 70) click.echo("Site Deployment Details") click.echo("=" * 70) click.echo(f"ID: {site.id}") click.echo(f"Site Name: {site.site_name}") click.echo(f"Custom Domain: {site.custom_hostname}") click.echo(f"\nStorage Zone:") click.echo(f" ID: {site.storage_zone_id}") click.echo(f" Name: {site.storage_zone_name}") click.echo(f" Region: {site.storage_zone_region}") click.echo(f" Password: {site.storage_zone_password}") click.echo(f"\nPull Zone:") click.echo(f" ID: {site.pull_zone_id}") click.echo(f" b-cdn Hostname: {site.pull_zone_bcdn_hostname}") click.echo(f"\nTimestamps:") click.echo(f" Created: {site.created_at.strftime('%Y-%m-%d %H:%M:%S')}") click.echo(f" Updated: {site.updated_at.strftime('%Y-%m-%d %H:%M:%S')}") click.echo("=" * 70) finally: session.close() except Exception as e: click.echo(f"Error getting site details: {e}", err=True) raise click.Abort() @app.command("remove-site") @click.option("--domain", prompt=True, help="Custom domain to remove") @click.option("--admin-user", help="Admin username for authentication") @click.option("--admin-password", help="Admin password for authentication") @click.confirmation_option(prompt="Are you sure you want to remove this site deployment record?") def remove_site(domain: str, admin_user: Optional[str], admin_password: Optional[str]): """Remove a site deployment record (requires admin)""" try: # Authenticate admin if not admin_user or not admin_password: admin_user, admin_password = prompt_admin_credentials() admin = authenticate_admin(admin_user, admin_password) if not admin: click.echo("Error: Authentication failed or insufficient permissions", err=True) raise click.Abort() # Remove site session = db_manager.get_session() try: deployment_repo = SiteDeploymentRepository(session) # Check if site exists site = deployment_repo.get_by_hostname(domain) if not site: click.echo(f"Error: Site with domain '{domain}' not found", err=True) raise click.Abort() # Delete the site success = deployment_repo.delete(site.id) if success: click.echo(f"Success: Site deployment record for '{domain}' has been removed") click.echo("\nNote: This does NOT delete resources from bunny.net.") click.echo("You must manually delete the Storage Zone and Pull Zone if needed.") else: click.echo(f"Error: Failed to remove site '{domain}'", err=True) raise click.Abort() finally: session.close() except Exception as e: click.echo(f"Error removing site: {e}", err=True) raise click.Abort() @app.command("sync-sites") @click.option("--admin-user", help="Admin username for authentication") @click.option("--admin-password", help="Admin password for authentication") @click.option("--dry-run", is_flag=True, help="Show what would be imported without making changes") def sync_sites(admin_user: Optional[str], admin_password: Optional[str], dry_run: bool): """Sync existing bunny.net sites with custom domains to database (requires admin)""" try: # Authenticate admin if not admin_user or not admin_password: admin_user, admin_password = prompt_admin_credentials() admin = authenticate_admin(admin_user, admin_password) if not admin: click.echo("Error: Authentication failed or insufficient permissions", err=True) raise click.Abort() # Get bunny.net API key try: api_key = get_bunny_account_api_key() except ValueError as e: click.echo(f"Error: {e}", err=True) click.echo("Please set BUNNY_ACCOUNT_API_KEY in your .env file", err=True) raise click.Abort() click.echo("\nSyncing sites from bunny.net...") if dry_run: click.echo("DRY RUN MODE - No changes will be made\n") # Initialize bunny.net client client = BunnyNetClient(api_key) session = db_manager.get_session() try: deployment_repo = SiteDeploymentRepository(session) # Get all storage zones (with passwords!) click.echo("Fetching Storage Zones from bunny.net...") storage_zones = client.get_storage_zones() storage_zone_map = {zone["Id"]: zone for zone in storage_zones} click.echo(f" Found {len(storage_zones)} Storage Zones") # Get all pull zones click.echo("Fetching Pull Zones from bunny.net...") pull_zones = client.get_pull_zones() click.echo(f" Found {len(pull_zones)} Pull Zones") # Process pull zones with custom hostnames imported = 0 skipped = 0 errors = 0 click.echo("\nProcessing Pull Zones with custom domains...") click.echo("=" * 80) for pz in pull_zones: # Skip if not linked to storage zone if not pz.get("StorageZoneId"): continue storage_zone_id = pz["StorageZoneId"] storage_zone = storage_zone_map.get(storage_zone_id) if not storage_zone: continue # Get pull zone details to see hostnames pz_details = client.get_pull_zone(pz["Id"]) if not pz_details: continue hostnames = pz_details.get("Hostnames", []) # Get the default b-cdn hostname default_hostname = next( (h["Value"] for h in hostnames if h.get("Value") and h["Value"].endswith(".b-cdn.net")), f"{pz['Name']}.b-cdn.net" ) # Filter for custom hostnames (not *.b-cdn.net) custom_hostnames = [ h["Value"] for h in hostnames if h.get("Value") and not h["Value"].endswith(".b-cdn.net") ] # Create list of sites to import: custom domains first, then bcdn-only if no custom domains sites_to_import = [] if custom_hostnames: for ch in custom_hostnames: sites_to_import.append((ch, default_hostname)) else: sites_to_import.append((None, default_hostname)) # Import each site deployment for custom_hostname, bcdn_hostname in sites_to_import: try: # Check if already exists check_hostname = custom_hostname or bcdn_hostname if deployment_repo.exists(check_hostname): click.echo(f"SKIP: {check_hostname} (already in database)") skipped += 1 continue if dry_run: click.echo(f"WOULD IMPORT: {check_hostname}") click.echo(f" Storage Zone: {storage_zone['Name']} (Region: {storage_zone.get('Region', 'Unknown')})") click.echo(f" Pull Zone: {pz['Name']} (ID: {pz['Id']})") click.echo(f" b-cdn Hostname: {bcdn_hostname}") if custom_hostname: click.echo(f" Custom Domain: {custom_hostname}") imported += 1 else: # Create site deployment deployment = deployment_repo.create( site_name=storage_zone['Name'], storage_zone_id=storage_zone['Id'], storage_zone_name=storage_zone['Name'], storage_zone_password=storage_zone.get('Password', ''), storage_zone_region=storage_zone.get('Region', ''), pull_zone_id=pz['Id'], pull_zone_bcdn_hostname=bcdn_hostname, custom_hostname=custom_hostname ) click.echo(f"IMPORTED: {check_hostname}") click.echo(f" Storage Zone: {storage_zone['Name']} (Region: {storage_zone.get('Region', 'Unknown')})") click.echo(f" Pull Zone: {pz['Name']} (ID: {pz['Id']})") if custom_hostname: click.echo(f" Custom Domain: {custom_hostname}") imported += 1 except Exception as e: click.echo(f"ERROR importing {check_hostname}: {e}", err=True) errors += 1 click.echo("=" * 80) click.echo(f"\nSync Summary:") click.echo(f" Imported: {imported}") click.echo(f" Skipped (already exists): {skipped}") click.echo(f" Errors: {errors}") if dry_run: click.echo("\nDRY RUN complete - no changes were made") click.echo("Run without --dry-run to import these sites") except BunnyNetAuthError as e: click.echo(f"Error: Authentication failed - {e}", err=True) click.echo("Please check your BUNNY_ACCOUNT_API_KEY", err=True) raise click.Abort() except BunnyNetAPIError as e: click.echo(f"Error: bunny.net API error - {e}", err=True) raise click.Abort() finally: session.close() except Exception as e: click.echo(f"Error syncing sites: {e}", err=True) raise click.Abort() @app.command() @click.option('--file', '-f', 'file_path', required=True, type=click.Path(exists=True), help='Path to CORA .xlsx file') @click.option('--name', '-n', required=True, help='Project name') @click.option('--money-site-url', '-m', help='Money site URL (e.g., https://example.com)') @click.option('--custom-anchors', '-a', help='Comma-separated list of custom anchor text (optional)') @click.option('--username', '-u', help='Username for authentication') @click.option('--password', '-p', help='Password for authentication') def ingest_cora(file_path: str, name: str, money_site_url: Optional[str], custom_anchors: Optional[str], username: Optional[str], password: Optional[str]): """Ingest a CORA .xlsx report and create a new project""" try: if not username or not password: username, password = prompt_admin_credentials() session = db_manager.get_session() try: user_repo = UserRepository(session) auth_service = AuthService(user_repo) user = auth_service.authenticate_user(username, password) if not user: click.echo("Error: Authentication failed", err=True) raise click.Abort() click.echo(f"Authenticated as: {user.username} ({user.role})") click.echo(f"\nParsing CORA file: {file_path}") custom_anchor_list = [] if custom_anchors: custom_anchor_list = [anchor.strip() for anchor in custom_anchors.split(',') if anchor.strip()] parser = CORAParser(file_path) cora_data = parser.parse(custom_anchor_text=custom_anchor_list) click.echo(f"Main Keyword: {cora_data['main_keyword']}") click.echo(f"Word Count: {cora_data['word_count']}") click.echo(f"Entities Found: {len(cora_data['entities'])}") click.echo(f"Related Searches: {len(cora_data['related_searches'])}") # Prompt for money_site_url if not provided if not money_site_url: money_site_url = click.prompt( "\nEnter money site URL (required for tiered linking)", type=str ) # Validate money_site_url if not money_site_url.startswith('http://') and not money_site_url.startswith('https://'): click.echo("Error: Money site URL must start with http:// or https://", err=True) raise click.Abort() # Clean up URL (remove trailing slash) money_site_url = money_site_url.rstrip('/') click.echo(f"\nCreating project: {name}") click.echo(f"Money Site URL: {money_site_url}") # Add money_site_url to cora_data cora_data['money_site_url'] = money_site_url project_repo = ProjectRepository(session) project = project_repo.create( user_id=user.id, name=name, data=cora_data ) click.echo(f"\nSuccess: Project '{project.name}' created (ID: {project.id})") click.echo(f"Main Keyword: {project.main_keyword}") click.echo(f"Money Site URL: {project.money_site_url}") click.echo(f"Entities: {len(project.entities or [])}") click.echo(f"Related Searches: {len(project.related_searches or [])}") if project.custom_anchor_text: click.echo(f"Custom Anchor Text: {', '.join(project.custom_anchor_text)}") except CORAParseError as e: click.echo(f"Error parsing CORA file: {e}", err=True) raise click.Abort() except ValueError as e: click.echo(f"Error creating project: {e}", err=True) raise click.Abort() finally: session.close() except Exception as e: click.echo(f"Error ingesting CORA file: {e}", err=True) raise click.Abort() @app.command() @click.option('--file', '-f', 'file_path', required=True, type=click.Path(exists=True), help='Path to simple .xlsx spreadsheet file') @click.option('--name', '-n', help='Project name (overrides project_name from spreadsheet if provided)') @click.option('--money-site-url', '-m', help='Money site URL (e.g., https://example.com)') @click.option('--username', '-u', help='Username for authentication') @click.option('--password', '-p', help='Password for authentication') def ingest_simple(file_path: str, name: Optional[str], money_site_url: Optional[str], username: Optional[str], password: Optional[str]): """Ingest a simple spreadsheet and create a new project Expected spreadsheet format: - First row: Headers (main_keyword, project_name, related_searches, entities) - Second row: Data values Required columns: main_keyword, project_name, related_searches, entities - main_keyword: Single phrase keyword - project_name: Name for the project - related_searches: Comma-delimited list (e.g., "term1, term2, term3") - entities: Comma-delimited list (e.g., "entity1, entity2, entity3") Optional columns (with defaults): - word_count: Default 1500 - term_frequency: Default 3 """ try: if not username or not password: username, password = prompt_admin_credentials() session = db_manager.get_session() try: user_repo = UserRepository(session) auth_service = AuthService(user_repo) user = auth_service.authenticate_user(username, password) if not user: click.echo("Error: Authentication failed", err=True) raise click.Abort() click.echo(f"Authenticated as: {user.username} ({user.role})") click.echo(f"\nParsing simple spreadsheet: {file_path}") parser = SimpleSpreadsheetParser(file_path) data = parser.parse() project_name = name or data.get("project_name") if not project_name: click.echo("Error: Project name is required (provide via --name or in spreadsheet)", err=True) raise click.Abort() click.echo(f"Main Keyword: {data['main_keyword']}") click.echo(f"Project Name: {project_name}") click.echo(f"Word Count: {data['word_count']}") click.echo(f"Term Frequency: {data['term_frequency']}") click.echo(f"Entities: {len(data['entities'])}") click.echo(f"Related Searches: {len(data['related_searches'])}") if data['entities']: click.echo(f" Entities: {', '.join(data['entities'][:5])}" + (f" ... (+{len(data['entities']) - 5} more)" if len(data['entities']) > 5 else "")) if data['related_searches']: click.echo(f" Related Searches: {', '.join(data['related_searches'][:5])}" + (f" ... (+{len(data['related_searches']) - 5} more)" if len(data['related_searches']) > 5 else "")) if not money_site_url: money_site_url = click.prompt( "\nEnter money site URL (required for tiered linking)", type=str ) if not money_site_url.startswith('http://') and not money_site_url.startswith('https://'): click.echo("Error: Money site URL must start with http:// or https://", err=True) raise click.Abort() money_site_url = money_site_url.rstrip('/') click.echo(f"\nCreating project: {project_name}") click.echo(f"Money Site URL: {money_site_url}") data['money_site_url'] = money_site_url project_data = {k: v for k, v in data.items() if k != 'project_name'} project_repo = ProjectRepository(session) project = project_repo.create( user_id=user.id, name=project_name, data=project_data ) click.echo(f"\nSuccess: Project '{project.name}' created (ID: {project.id})") click.echo(f"Main Keyword: {project.main_keyword}") click.echo(f"Money Site URL: {project.money_site_url}") click.echo(f"Word Count: {project.word_count}") click.echo(f"Term Frequency: {project.term_frequency}") click.echo(f"Entities: {len(project.entities or [])}") click.echo(f"Related Searches: {len(project.related_searches or [])}") except CORAParseError as e: click.echo(f"Error parsing spreadsheet: {e}", err=True) raise click.Abort() except ValueError as e: click.echo(f"Error creating project: {e}", err=True) raise click.Abort() finally: session.close() except Exception as e: click.echo(f"Error ingesting spreadsheet: {e}", err=True) raise click.Abort() @app.command() @click.option('--username', '-u', help='Username for authentication') @click.option('--password', '-p', help='Password for authentication') def list_projects(username: Optional[str], password: Optional[str]): """List all projects for the authenticated user""" try: if not username or not password: username, password = prompt_admin_credentials() session = db_manager.get_session() try: user_repo = UserRepository(session) auth_service = AuthService(user_repo) user = auth_service.authenticate_user(username, password) if not user: click.echo("Error: Authentication failed", err=True) raise click.Abort() project_repo = ProjectRepository(session) if user.is_admin(): projects = project_repo.get_all() click.echo(f"\nAll Projects (Admin View):") else: projects = project_repo.get_by_user_id(user.id) click.echo(f"\nYour Projects:") if not projects: click.echo("No projects found") return click.echo(f"Total projects: {len(projects)}") click.echo("-" * 80) click.echo(f"{'ID':<5} {'Name':<30} {'Keyword':<25} {'Created':<20}") click.echo("-" * 80) for project in projects: created_str = project.created_at.strftime('%Y-%m-%d %H:%M:%S') click.echo(f"{project.id:<5} {project.name[:29]:<30} {project.main_keyword[:24]:<25} {created_str:<20}") click.echo("-" * 80) finally: session.close() except Exception as e: click.echo(f"Error listing projects: {e}", err=True) raise click.Abort() @app.command("generate-batch") @click.option('--job-file', '-j', required=True, type=click.Path(exists=True), help='Path to job JSON file') @click.option('--username', '-u', help='Username for authentication') @click.option('--password', '-p', help='Password for authentication') @click.option('--debug', is_flag=True, help='Save AI responses to debug_output/') @click.option('--continue-on-error', is_flag=True, help='Continue processing if article generation fails') @click.option('--model', '-m', default='gpt-4o-mini', help='AI model to use (gpt-4o-mini, x-ai/grok-4-fast)') def generate_batch( job_file: str, username: Optional[str], password: Optional[str], debug: bool, continue_on_error: bool, model: str ): """Generate content batch from job file""" try: if not username or not password: username, password = prompt_admin_credentials() session = db_manager.get_session() try: user_repo = UserRepository(session) auth_service = AuthService(user_repo) user = auth_service.authenticate_user(username, password) if not user: click.echo("Error: Authentication failed", err=True) raise click.Abort() click.echo(f"Authenticated as: {user.username} ({user.role})") api_key = os.getenv("OPENROUTER_API_KEY") if not api_key: click.echo("Error: OPENROUTER_API_KEY not found in environment", err=True) click.echo("Please set OPENROUTER_API_KEY in your .env file", err=True) raise click.Abort() from src.generation.job_config import JobConfig job_config = JobConfig(job_file) jobs = job_config.get_jobs() has_models_in_job = any(job.models is not None for job in jobs) if has_models_in_job and model != 'gpt-4o-mini': click.echo(f"Warning: Job file contains per-stage model configuration.") click.echo(f" The --model flag will be ignored in favor of job config.\n") click.echo(f"Initializing AI client with default model: {model}") ai_client = AIClient(api_key=api_key, model=model) prompt_manager = PromptManager() project_repo = ProjectRepository(session) content_repo = GeneratedContentRepository(session) site_deployment_repo = SiteDeploymentRepository(session) content_generator = ContentGenerator( ai_client=ai_client, prompt_manager=prompt_manager, project_repo=project_repo, content_repo=content_repo ) max_workers = get_concurrent_workers() job_max_workers = jobs[0].max_workers if jobs and jobs[0].max_workers else None final_max_workers = job_max_workers or max_workers batch_processor = BatchProcessor( content_generator=content_generator, content_repo=content_repo, project_repo=project_repo, site_deployment_repo=site_deployment_repo, max_workers=final_max_workers ) click.echo(f"\nProcessing job file: {job_file}") click.echo(f"Concurrent workers: {final_max_workers}") if debug: click.echo("Debug mode: AI responses will be saved to debug_output/\n") batch_processor.process_job( job_file_path=job_file, debug=debug, continue_on_error=continue_on_error ) finally: session.close() except Exception as e: click.echo(f"Error processing batch: {e}", err=True) raise click.Abort() @app.command("deploy-batch") @click.option('--batch-id', '-b', required=True, type=int, help='Project/batch ID to deploy') @click.option('--username', '-u', help='Username for authentication') @click.option('--password', '-p', help='Password for authentication') @click.option('--continue-on-error', is_flag=True, default=True, help='Continue if file fails (default: True)') @click.option('--dry-run', is_flag=True, help='Preview what would be deployed') def deploy_batch( batch_id: int, username: Optional[str], password: Optional[str], continue_on_error: bool, dry_run: bool ): """Deploy all content in a batch to cloud storage""" try: if not username or not password: username, password = prompt_admin_credentials() admin = authenticate_admin(username, password) if not admin: click.echo("Error: Authentication failed or insufficient permissions", err=True) raise click.Abort() click.echo(f"Authenticated as: {admin.username} ({admin.role})") session = db_manager.get_session() try: project_repo = ProjectRepository(session) content_repo = GeneratedContentRepository(session) site_repo = SiteDeploymentRepository(session) page_repo = SitePageRepository(session) project = project_repo.get_by_id(batch_id) if not project: click.echo(f"Error: Project/batch {batch_id} not found", err=True) raise click.Abort() click.echo(f"\nDeploying batch: {project.name} (ID: {batch_id})") click.echo(f"Keyword: {project.main_keyword}") articles = content_repo.get_by_project_id(batch_id) click.echo(f"Found {len(articles)} articles") if dry_run: click.echo("\nDRY RUN MODE - No files will be uploaded\n") for article in articles: if not article.site_deployment_id: click.echo(f"SKIP: Article {article.id} - No site assigned") continue site = site_repo.get_by_id(article.site_deployment_id) if not site: click.echo(f"SKIP: Article {article.id} - Site not found") continue from src.generation.url_generator import generate_file_path, generate_public_url file_path = generate_file_path(article) url = generate_public_url(site, file_path) click.echo(f"WOULD DEPLOY: {article.title[:50]}") click.echo(f" File: {file_path}") click.echo(f" URL: {url}") site_ids = set(a.site_deployment_id for a in articles if a.site_deployment_id) for site_id in site_ids: pages = page_repo.get_by_site(site_id) for page in pages: click.echo(f"WOULD DEPLOY: {page.page_type}.html") click.echo("\nDry run complete. Use without --dry-run to actually deploy.") return storage_client = BunnyStorageClient(max_retries=3) url_logger = URLLogger() deployment_service = DeploymentService( storage_client=storage_client, content_repo=content_repo, site_repo=site_repo, page_repo=page_repo, url_logger=url_logger ) click.echo(f"\nStarting deployment...") click.echo(f"Continue on error: {continue_on_error}") click.echo("") results = deployment_service.deploy_batch( project_id=batch_id, continue_on_error=continue_on_error ) click.echo("\n" + "=" * 70) click.echo("Deployment Summary") click.echo("=" * 70) click.echo(f"Articles deployed: {results['articles_deployed']}") click.echo(f"Articles failed: {results['articles_failed']}") click.echo(f"Pages deployed: {results['pages_deployed']}") click.echo(f"Pages failed: {results['pages_failed']}") click.echo(f"Total time: {results['total_time']:.1f}s") if results['errors']: click.echo("\nErrors:") for error in results['errors']: if error['type'] == 'article': click.echo(f" Article {error['id']} ({error.get('title', 'N/A')[:40]}): {error['error']}") else: click.echo(f" Page {error.get('page_type', 'N/A')} (Site {error.get('site_id')}): {error['error']}") click.echo("=" * 70) if results['articles_failed'] > 0 or results['pages_failed'] > 0: raise click.Abort() except BunnyStorageError as e: click.echo(f"\nError: Storage upload failed - {e}", err=True) raise click.Abort() finally: session.close() except Exception as e: click.echo(f"Error deploying batch: {e}", err=True) raise click.Abort() @app.command("verify-deployment") @click.option('--batch-id', '-b', required=True, type=int, help='Project/batch ID to verify') @click.option('--sample', '-s', type=int, help='Number of random URLs to check (default: check all)') @click.option('--timeout', '-t', type=int, default=10, help='Request timeout in seconds (default: 10)') def verify_deployment(batch_id: int, sample: Optional[int], timeout: int): """Verify deployed URLs return 200 OK status""" try: session = db_manager.get_session() try: content_repo = GeneratedContentRepository(session) project_repo = ProjectRepository(session) project = project_repo.get_by_id(batch_id) if not project: click.echo(f"Error: Project/batch {batch_id} not found", err=True) raise click.Abort() click.echo(f"Verifying deployment for batch: {project.name} (ID: {batch_id})") click.echo(f"Keyword: {project.main_keyword}\n") articles = content_repo.get_by_project_id(batch_id) deployed_articles = [a for a in articles if a.deployed_url and a.status == 'deployed'] if not deployed_articles: click.echo("No deployed articles found for this batch.") return click.echo(f"Found {len(deployed_articles)} deployed articles") urls_to_check = deployed_articles if sample and sample < len(deployed_articles): urls_to_check = random.sample(deployed_articles, sample) click.echo(f"Checking random sample of {sample} URLs\n") else: click.echo(f"Checking all {len(deployed_articles)} URLs\n") successful = [] failed = [] for article in urls_to_check: url = article.deployed_url try: response = requests.get(url, timeout=timeout, allow_redirects=True) if response.status_code == 200: successful.append((url, article.title)) click.echo(f"✓ {url}") else: failed.append((url, article.title, response.status_code)) click.echo(f"✗ {url} (HTTP {response.status_code})") except requests.exceptions.RequestException as e: failed.append((url, article.title, str(e))) click.echo(f"✗ {url} (Error: {type(e).__name__})") click.echo("\n" + "=" * 70) click.echo("Verification Summary") click.echo("=" * 70) click.echo(f"Total checked: {len(urls_to_check)}") click.echo(f"Successful: {len(successful)}") click.echo(f"Failed: {len(failed)}") if failed: click.echo("\nFailed URLs:") for url, title, error in failed: title_preview = title[:50] + "..." if len(title) > 50 else title click.echo(f" {url}") click.echo(f" Title: {title_preview}") click.echo(f" Error: {error}") click.echo("=" * 70) if failed: raise click.Abort() finally: session.close() except Exception as e: click.echo(f"Error verifying deployment: {e}", err=True) raise click.Abort() @app.command("get-links") @click.option('--project-id', '-p', required=True, type=int, help='Project ID to get links for') @click.option('--tier', '-t', required=True, help='Tier to filter (e.g., "1" or "2+" for tier 2 and above)') @click.option('--with-anchor-text', is_flag=True, help='Include anchor text used for tiered links') @click.option('--with-destination-url', is_flag=True, help='Include destination URL that the article links to') def get_links(project_id: int, tier: str, with_anchor_text: bool, with_destination_url: bool): """Export article URLs with optional link details for a project and tier""" import csv import sys from src.database.repositories import ArticleLinkRepository try: session = db_manager.get_session() try: content_repo = GeneratedContentRepository(session) project_repo = ProjectRepository(session) link_repo = ArticleLinkRepository(session) project = project_repo.get_by_id(project_id) if not project: click.echo(f"Error: Project {project_id} not found", err=True) raise click.Abort() tier_range_mode = False min_tier = 1 if tier.endswith('+'): tier_range_mode = True try: min_tier = int(tier[:-1]) except ValueError: click.echo(f"Error: Invalid tier format '{tier}'. Use '1', '2', or '2+'", err=True) raise click.Abort() else: try: min_tier = int(tier) tier_range_mode = False except ValueError: click.echo(f"Error: Invalid tier format '{tier}'. Use '1', '2', or '2+'", err=True) raise click.Abort() all_articles = content_repo.get_by_project_id(project_id) if tier_range_mode: articles = [a for a in all_articles if a.deployed_url and int(a.tier.replace('tier', '')) >= min_tier] else: tier_str = f"tier{min_tier}" articles = [a for a in all_articles if a.deployed_url and a.tier == tier_str] if not articles: click.echo(f"No deployed articles found for project {project_id} with tier filter '{tier}'", err=True) raise click.Abort() csv_writer = csv.writer(sys.stdout) header = ['article_url', 'tier', 'title'] if with_anchor_text: header.append('anchor_text') if with_destination_url: header.append('destination_url') csv_writer.writerow(header) for article in articles: row = [article.deployed_url, article.tier, article.title] if with_anchor_text or with_destination_url: tiered_links = link_repo.get_by_source_article(article.id) tiered_links = [link for link in tiered_links if link.link_type == 'tiered'] if tiered_links: for link in tiered_links: row_with_link = row.copy() if with_anchor_text: row_with_link.append(link.anchor_text or '') if with_destination_url: if link.to_url: row_with_link.append(link.to_url) elif link.to_content_id: target_article = content_repo.get_by_id(link.to_content_id) row_with_link.append(target_article.deployed_url if target_article and target_article.deployed_url else '') else: row_with_link.append('') csv_writer.writerow(row_with_link) else: if with_anchor_text: row.append('') if with_destination_url: row.append('') csv_writer.writerow(row) else: csv_writer.writerow(row) finally: session.close() except Exception as e: click.echo(f"Error getting links: {e}", err=True) raise click.Abort() if __name__ == "__main__": app()