tags. Do NOT include ,
, or tags." +} +``` + +**content_augmentation.json**: +```json +{ + "system_message": "You are an expert content editor...", + "user_prompt": "Please expand on the following article to add more detail and depth, ensuring you maintain the existing topical focus. Target word count: {target_word_count}\n\nCurrent article:\n{content}\n\nReturn the expanded article as an HTML fragment." +} +``` + +--- + +### Task 2.3: Create PromptManager +**File**: `src/generation/ai_client.py` (add to same file) + +```python +class PromptManager: + def __init__(self, prompts_dir: str = "src/generation/prompts"): + self.prompts_dir = prompts_dir + self.prompts = {} + + def load_prompt(self, prompt_name: str) -> dict: + """Load prompt from JSON file""" + pass + + def format_prompt(self, prompt_name: str, **kwargs) -> tuple[str, str]: + """ + Format prompt with variables + Returns: (system_message, user_prompt) + """ + pass +``` + +--- + +## **PHASE 3: Core Generation Pipeline** + +### Task 3.1: Implement ContentGenerator Service +**File**: `src/generation/service.py` + +```python +class ContentGenerator: + def __init__( + self, + ai_client: AIClient, + prompt_manager: PromptManager, + project_repo: ProjectRepository, + content_repo: GeneratedContentRepository + ): + self.ai_client = ai_client + self.prompt_manager = prompt_manager + self.project_repo = project_repo + self.content_repo = content_repo +``` + +--- + +### Task 3.2: Implement Stage 1 - Title Generation +**File**: `src/generation/service.py` + +```python +def generate_title(self, project_id: int, debug: bool = False) -> str: + """ + Generate SEO-optimized title + + Returns: title string + Saves to debug_output/title_project_{id}_{timestamp}.txt if debug=True + """ + # Fetch project + # Load prompt + # Call AI + # If debug: save response to debug_output/ + # Return title + pass +``` + +--- + +### Task 3.3: Implement Stage 2 - Outline Generation +**File**: `src/generation/service.py` + +```python +def generate_outline( + self, + project_id: int, + title: str, + min_h2: int, + max_h2: int, + min_h3: int, + max_h3: int, + debug: bool = False +) -> dict: + """ + Generate article outline in JSON format + + Returns: {"outline": [{"h2": "...", "h3": ["...", "..."]}]} + + Uses json_mode=True in AI call to ensure JSON response + Validates: at least min_h2 headings, at least min_h3 total subheadings + Saves to debug_output/outline_project_{id}_{timestamp}.json if debug=True + """ + pass +``` + +**Validation**: +- Parse JSON response +- Count h2 tags (must be >= min_h2) +- Count total h3 tags across all h2s (must be >= min_h3) +- Raise error if validation fails + +--- + +### Task 3.4: Implement Stage 3 - Content Generation +**File**: `src/generation/service.py` + +```python +def generate_content( + self, + project_id: int, + title: str, + outline: dict, + debug: bool = False +) -> str: + """ + Generate full article HTML fragment + + Returns: HTML string withtags + Does NOT include ,
, or tags + + Saves to debug_output/content_project_{id}_{timestamp}.html if debug=True + """ + pass +``` + +**HTML fragment format**: +```html +Paragraph content...
+More content...
+``` + +--- + +### Task 3.5: Implement Word Count Validation +**File**: `src/generation/service.py` + +```python +def validate_word_count(self, content: str, min_words: int, max_words: int) -> tuple[bool, int]: + """ + Validate content word count + + Returns: (is_valid, actual_count) + - is_valid: True if min_words <= actual_count <= max_words + - actual_count: number of words in content + + Implementation: Strip HTML tags, split on whitespace, count tokens + """ + pass +``` + +--- + +### Task 3.6: Implement Simple Augmentation +**File**: `src/generation/service.py` + +```python +def augment_content( + self, + content: str, + target_word_count: int, + debug: bool = False +) -> str: + """ + Expand article content to meet minimum word count + + Called ONLY if word_count < min_word_count + Makes ONE API call only + + Saves to debug_output/augmented_project_{id}_{timestamp}.html if debug=True + """ + pass +``` + +--- + +## **PHASE 4: Batch Processing** + +### Task 4.1: Create JobConfig Parser +**File**: `src/generation/job_config.py` + +```python +from dataclasses import dataclass +from typing import Optional + +TIER_DEFAULTS = { + "tier1": { + "min_word_count": 2000, + "max_word_count": 2500, + "min_h2_tags": 3, + "max_h2_tags": 5, + "min_h3_tags": 5, + "max_h3_tags": 10 + }, + "tier2": { + "min_word_count": 1500, + "max_word_count": 2000, + "min_h2_tags": 2, + "max_h2_tags": 4, + "min_h3_tags": 3, + "max_h3_tags": 8 + }, + "tier3": { + "min_word_count": 1000, + "max_word_count": 1500, + "min_h2_tags": 2, + "max_h2_tags": 3, + "min_h3_tags": 2, + "max_h3_tags": 6 + } +} + +@dataclass +class TierConfig: + count: int + min_word_count: int + max_word_count: int + min_h2_tags: int + max_h2_tags: int + min_h3_tags: int + max_h3_tags: int + +@dataclass +class Job: + project_id: int + tiers: dict[str, TierConfig] + +class JobConfig: + def __init__(self, job_file_path: str): + """Load and parse job file, apply defaults""" + pass + + def get_jobs(self) -> list[Job]: + """Return list of all jobs in file""" + pass + + def get_tier_config(self, job: Job, tier_name: str) -> Optional[TierConfig]: + """Get tier config with defaults applied""" + pass +``` + +--- + +### Task 4.2: Create BatchProcessor +**File**: `src/generation/batch_processor.py` + +```python +class BatchProcessor: + def __init__( + self, + content_generator: ContentGenerator, + content_repo: GeneratedContentRepository, + project_repo: ProjectRepository + ): + pass + + def process_job( + self, + job_file_path: str, + debug: bool = False, + continue_on_error: bool = False + ): + """ + Process all jobs in job file + + For each job: + For each tier: + For count times: + 1. Generate title (log to console) + 2. Generate outline + 3. Generate content + 4. Validate word count + 5. If below min, augment once + 6. Save to GeneratedContent table + + Logs progress to console + If debug=True, saves AI responses to debug_output/ + """ + pass +``` + +**Console output format**: +``` +Processing Job 1/3: Project ID 5 + Tier 1: Generating 5 articles + [1/5] Generating title... "Ultimate Guide to SEO in 2025" + [1/5] Generating outline... 4 H2s, 8 H3s + [1/5] Generating content... 1,845 words + [1/5] Below minimum (2000), augmenting... 2,123 words + [1/5] Saved (ID: 42, Status: augmented) + [2/5] Generating title... "Advanced SEO Techniques" + ... + Tier 2: Generating 10 articles + ... + +Summary: + Jobs processed: 3/3 + Articles generated: 45/45 + Augmented: 12 + Failed: 0 +``` + +--- + +### Task 4.3: Error Handling & Retry Logic +**File**: `src/generation/batch_processor.py` + +**Error handling strategy**: +- AI API errors: Log error, mark as `status='failed'`, save to DB +- If `continue_on_error=True`: continue to next article +- If `continue_on_error=False`: stop batch processing +- Database errors: Always abort (data integrity) +- Invalid job file: Fail fast with validation error + +**Retry logic** (in AIClient): +- Network errors: 3 retries with exponential backoff (1s, 2s, 4s) +- Rate limit errors: Respect Retry-After header +- Other errors: No retry, raise immediately + +--- + +## **PHASE 5: CLI Integration** + +### Task 5.1: Add generate-batch Command +**File**: `src/cli/commands.py` + +```python +@app.command("generate-batch") +@click.option('--job-file', '-j', required=True, type=click.Path(exists=True), + help='Path to job JSON file') +@click.option('--username', '-u', help='Username for authentication') +@click.option('--password', '-p', help='Password for authentication') +@click.option('--debug', is_flag=True, help='Save AI responses to debug_output/') +@click.option('--continue-on-error', is_flag=True, + help='Continue processing if article generation fails') +@click.option('--model', '-m', default='gpt-4o-mini', + help='AI model to use (gpt-4o-mini, claude-sonnet-4.5)') +def generate_batch( + job_file: str, + username: Optional[str], + password: Optional[str], + debug: bool, + continue_on_error: bool, + model: str +): + """Generate content batch from job file""" + # Authenticate user + # Initialize AIClient with OpenRouter + # Initialize PromptManager, ContentGenerator, BatchProcessor + # Call process_job() + # Show summary + pass +``` + +--- + +### Task 5.2: Add Progress Logging & Debug Output +**File**: `src/generation/batch_processor.py` + +**Debug output** (when `--debug` flag used): +- Create `debug_output/` directory if not exists +- For each AI call, save response to file: + - `debug_output/title_project{id}_tier{tier}_{n}_{timestamp}.txt` + - `debug_output/outline_project{id}_tier{tier}_{n}_{timestamp}.json` + - `debug_output/content_project{id}_tier{tier}_{n}_{timestamp}.html` + - `debug_output/augmented_project{id}_tier{tier}_{n}_{timestamp}.html` +- Also echo to console with `click.echo()` + +**Normal output** (without `--debug`): +- Always show title when generated: `"Generated title: {title}"` +- Show word counts and status +- Show progress counter `[n/total]` + +--- + +## **PHASE 6: Testing & Validation** + +### Task 6.1: Create Unit Tests + +#### `tests/unit/test_ai_client.py` +```python +def test_generate_completion_success(): + """Test successful AI completion""" + pass + +def test_generate_completion_json_mode(): + """Test JSON mode returns valid JSON""" + pass + +def test_generate_completion_retry_on_network_error(): + """Test retry logic for network errors""" + pass +``` + +#### `tests/unit/test_content_generator.py` +```python +def test_generate_title(): + """Test title generation with mocked AI response""" + pass + +def test_generate_outline_valid_structure(): + """Test outline generation returns valid JSON with min h2/h3""" + pass + +def test_generate_content_html_fragment(): + """Test content is HTML fragment (no tag)""" + pass + +def test_validate_word_count(): + """Test word count validation with various HTML inputs""" + pass + +def test_augment_content_called_once(): + """Test augmentation only called once""" + pass +``` + +#### `tests/unit/test_job_config.py` +```python +def test_load_job_config_valid(): + """Test loading valid job file""" + pass + +def test_tier_defaults_applied(): + """Test defaults applied when not in job file""" + pass + +def test_multiple_jobs_in_file(): + """Test parsing file with multiple jobs""" + pass +``` + +#### `tests/unit/test_batch_processor.py` +```python +def test_process_job_success(): + """Test successful batch processing""" + pass + +def test_process_job_with_augmentation(): + """Test articles below min word count are augmented""" + pass + +def test_process_job_continue_on_error(): + """Test continue_on_error flag behavior""" + pass +``` + +--- + +### Task 6.2: Create Integration Test +**File**: `tests/integration/test_generate_batch.py` + +```python +def test_generate_batch_end_to_end(test_db, mock_ai_client): + """ + End-to-end test: + 1. Create test project in DB + 2. Create test job file + 3. Run batch processor + 4. Verify GeneratedContent records created + 5. Verify word counts within range + 6. Verify HTML structure + """ + pass +``` + +--- + +### Task 6.3: Create Example Job Files + +#### `jobs/example_tier1_batch.json` +```json +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 5 + } + } + } + ] +} +``` +(Uses all defaults for tier1) + +#### `jobs/example_multi_tier_batch.json` +```json +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 5, + "min_word_count": 2200, + "max_word_count": 2600 + }, + "tier2": { + "count": 10 + }, + "tier3": { + "count": 15, + "max_h2_tags": 4 + } + } + }, + { + "project_id": 2, + "tiers": { + "tier1": { + "count": 3 + } + } + } + ] +} +``` + +#### `jobs/README.md` +Document job file format and examples + +--- + +## **PHASE 7: Cleanup & Deprecation** + +### Task 7.1: Remove Old ContentRuleEngine +**Action**: Delete or gut `src/generation/rule_engine.py` + +Only keep if it has reusable utilities. Otherwise remove entirely. + +--- + +### Task 7.2: Remove Old Validator Logic +**Action**: Review `src/generation/validator.py` (if exists) + +Remove any strict CORA validation beyond word count. Keep only simple validation utilities. + +--- + +### Task 7.3: Update Documentation +**Files to update**: +- `docs/stories/story-2.2. simplified-ai-content-generation.md` - Status to "In Progress" → "Done" +- `docs/architecture/workflows.md` - Document simplified generation flow +- `docs/architecture/components.md` - Update generation component description + +--- + +## Implementation Order Recommendation + +1. **Phase 1** (Data Layer) - Required foundation +2. **Phase 2** (AI Client) - Required for generation +3. **Phase 3** (Core Logic) - Implement one stage at a time, test each +4. **Phase 4** (Batch Processing) - Orchestrate stages +5. **Phase 5** (CLI) - Make accessible to users +6. **Phase 6** (Testing) - Can be done in parallel with implementation +7. **Phase 7** (Cleanup) - Final polish + +**Estimated effort**: +- Phase 1-2: 4-6 hours +- Phase 3: 6-8 hours +- Phase 4: 3-4 hours +- Phase 5: 2-3 hours +- Phase 6: 4-6 hours +- Phase 7: 1-2 hours +- **Total**: 20-29 hours + +--- + +## Critical Dev Notes + +### OpenRouter Specifics +- API key from environment: `OPENROUTER_API_KEY` +- Model format: `"provider/model-name"` +- Supports OpenAI SDK drop-in replacement +- Rate limits vary by model (check OpenRouter docs) + +### HTML Fragment Format +Content generation returns HTML like: +```html +Introduction paragraph with relevant keywords and entities.
+Detailed content about subtopic.
+More detailed content.
+Content continues...
+``` + +**No document structure**: No ``, ``, ``, or `` tags. + +### Word Count Method +```python +import re +from html import unescape + +def count_words(html_content: str) -> int: + # Strip HTML tags + text = re.sub(r'<[^>]+>', '', html_content) + # Unescape HTML entities + text = unescape(text) + # Split and count + words = text.split() + return len(words) +``` + +### Debug Output Directory +- Create `debug_output/` at project root if not exists +- Add to `.gitignore` +- Filename format: `{stage}_project{id}_tier{tier}_article{n}_{timestamp}.{ext}` +- Example: `title_project5_tier1_article3_20251020_143022.txt` + +### Tier Constants Location +Define in `src/generation/job_config.py` as module-level constant for easy reference. + +### Future Extensibility +Job file structure designed to support: +- Custom interlinking rules (Story 2.4+) +- Template selection (Story 3.x) +- Deployment targets (Story 4.x) +- SEO metadata overrides + +Keep job parsing flexible to add new fields without breaking existing jobs. + +--- + +## Testing Strategy + +### Unit Test Mocking +Mock `AIClient.generate_completion()` to return realistic HTML: +```python +@pytest.fixture +def mock_title_response(): + return "The Ultimate Guide to Sustainable Gardening in 2025" + +@pytest.fixture +def mock_outline_response(): + return { + "outline": [ + {"h2": "Getting Started", "h3": ["Tools", "Planning"]}, + {"h2": "Best Practices", "h3": ["Watering", "Composting"]} + ] + } + +@pytest.fixture +def mock_content_response(): + return """Sustainable gardening begins with proper planning...
+Essential tools include...
""" +``` + +### Integration Test Database +Use `conftest.py` fixture with in-memory SQLite and test data: +```python +@pytest.fixture +def test_project(test_db): + project_repo = ProjectRepository(test_db) + return project_repo.create( + user_id=1, + name="Test Project", + data={ + "main_keyword": "sustainable gardening", + "entities": ["composting", "organic soil"], + "related_searches": ["how to compost", "organic gardening tips"] + } + ) +``` + +--- + +## Success Criteria + +Story is complete when: +1. All database models and repositories implemented +2. AIClient successfully calls OpenRouter API +3. Three-stage generation pipeline works end-to-end +4. Batch processor handles multiple jobs/tiers +5. CLI command `generate-batch` functional +6. Debug output saves to `debug_output/` when `--debug` used +7. All unit tests pass +8. Integration test demonstrates full workflow +9. Example job files work correctly +10. Documentation updated + +**Acceptance**: Run `generate-batch` on real project, verify content saved to database with correct word count and structure. + diff --git a/docs/stories/story-2.2. simplified-ai-content-generation.md b/docs/stories/story-2.2. simplified-ai-content-generation.md new file mode 100644 index 0000000..356c4ac --- /dev/null +++ b/docs/stories/story-2.2. simplified-ai-content-generation.md @@ -0,0 +1,40 @@ +# Story 2.2: Simplified AI Content Generation via Batch Job + +## Status +Completed + +## Story +**As a** User, +**I want** to control AI content generation via a batch file that specifies word count and heading limits, +**so that** I can easily create topically relevant articles without unnecessary complexity or rigid validation. + +## Acceptance Criteria +1. **Batch Job Control:** The `generate-batch` command accepts a JSON job file that specifies `min_word_count`, `max_word_count`, `max_h2_tags`, and `max_h3_tags` for each tier. +2. **Three-Stage Generation:** The system uses a simple three-stage pipeline: + * Generates a title using the project's SEO data. + * Generates an outline based on the title, SEO data, and the `max_h2`/`max_h3` limits from the job file. + * Generates the full article content based on the validated outline. +3. **SEO Data Integration:** The generation process for all stages is informed by the project's `keyword`, `entities`, and `related_searches` to ensure topical relevance. +4. **Word Count Validation:** After generation, the system validates the content *only* against the `min_word_count` and `max_word_count` specified in the job file. +5. **Simple Augmentation:** If the generated content is below `min_word_count`, the system makes **one** attempt to append additional content using a simple "expand on this article" prompt. +6. **Database Storage:** The final generated title, outline, and content are stored in the `GeneratedContent` table. +7. **CLI Execution:** The `generate-batch` command successfully runs the job, logs progress to the console, and indicates when the process is complete. + +## Dev Notes +* **Objective:** This story replaces the previous, overly complex stories 2.2 and 2.3. The goal is maximum simplicity and user control via the job file. +* **Key Change:** Remove the entire `ContentRuleEngine` and all strict CORA validation logic. The only validation required is a final word count check. +* **Job File is King:** All operational parameters (`min_word_count`, `max_word_count`, `max_h2_tags`, `max_h3_tags`) must be read from the job file for each tier being processed. +* **Augmentation:** Keep it simple. If `word_count < min_word_count`, make a single API call to the AI with a prompt like: "Please expand on the following article to add more detail and depth, ensuring you maintain the existing topical focus. Here is the article: {content}". Do not create a complex augmentation system. + +## Implementation Plan + +See **[story-2.2-task-breakdown.md](story-2.2-task-breakdown.md)** for detailed implementation tasks. + +The task breakdown is organized into 7 phases: +1. **Phase 1**: Data Model & Schema Design (GeneratedContent table, repositories, job file schema) +2. **Phase 2**: AI Client & Prompt Management (OpenRouter integration, prompt templates) +3. **Phase 3**: Core Generation Pipeline (title, outline, content generation with validation) +4. **Phase 4**: Batch Processing (job config parser, batch processor, error handling) +5. **Phase 5**: CLI Integration (generate-batch command, progress logging, debug output) +6. **Phase 6**: Testing & Validation (unit tests, integration tests, example job files) +7. **Phase 7**: Cleanup & Deprecation (remove old rule engine and validators) diff --git a/env.example b/env.example index 2cceb78..2585341 100644 --- a/env.example +++ b/env.example @@ -2,7 +2,7 @@ DATABASE_URL=sqlite:///./content_automation.db # AI Service Configuration (OpenRouter) -AI_API_KEY=your_openrouter_api_key_here +OPENROUTER_API_KEY=your_openrouter_api_key_here AI_API_BASE_URL=https://openrouter.ai/api/v1 AI_MODEL=anthropic/claude-3.5-sonnet diff --git a/et --hard d81537f b/et --hard d81537f new file mode 100644 index 0000000..69d12e7 --- /dev/null +++ b/et --hard d81537f @@ -0,0 +1,16 @@ +[33m5b5bd1b[m[33m ([m[1;36mHEAD[m[33m -> [m[1;32mfeature/tier-word-count-override[m[33m)[m Add tier-specific word count and outline controls +[33m3063fc4[m[33m ([m[1;31morigin/main[m[33m, [m[1;31morigin/HEAD[m[33m, [m[1;32mmain[m[33m)[m Story 2.3 - content generation script nightmare alomst done - fixed (maybe) outline too big issue +[33mb6b0acf[m Story 2.3 - content generation script nightmare alomst done - pre-fix outline too big issue +[33mf73b070[m[33m ([m[1;31mgithub/main[m[33m)[m Story 2.3 - content generation script finished - fix ci +[33me2afabb[m Story 2.3 - content generation script finished +[33m0069e6e[m Story 2.2 - rule engine finished +[33md81537f[m Story 2.1 finished +[33m02dd5a3[m Story 2.1 finished +[33m29ecaec[m Story 1.7 finished +[33mda797c2[m Story 1.6 finished - added sync +[33m4cada9d[m Story 1.6 finished +[33mb6e495e[m feat: Story 1.5 - CLI User Management +[33m0a223e2[m Complete Story 1.4: Internal API Foundation +[33m8641bca[m Complete Epic 1 Stories 1.1-1.3: Foundation, Database, and Authentication +[33m70b9de2[m feat: Complete Story 1.1 - Project Initialization & Configuration +[33m31b9580[m Initial commit: Project structure and planning documents diff --git a/jobs/README.md b/jobs/README.md new file mode 100644 index 0000000..633d42c --- /dev/null +++ b/jobs/README.md @@ -0,0 +1,179 @@ +# Job File Format + +Job files define batch content generation parameters using JSON format. + +## Structure + +```json +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 5, + "min_word_count": 2000, + "max_word_count": 2500, + "min_h2_tags": 3, + "max_h2_tags": 5, + "min_h3_tags": 5, + "max_h3_tags": 10 + } + } + } + ] +} +``` + +## Fields + +### Job Level +- `project_id` (required): The project ID to generate content for +- `tiers` (required): Dictionary of tier configurations + +### Tier Level +- `count` (required): Number of articles to generate for this tier +- `min_word_count` (optional): Minimum word count (uses defaults if not specified) +- `max_word_count` (optional): Maximum word count (uses defaults if not specified) +- `min_h2_tags` (optional): Minimum H2 headings (uses defaults if not specified) +- `max_h2_tags` (optional): Maximum H2 headings (uses defaults if not specified) +- `min_h3_tags` (optional): Minimum H3 subheadings total (uses defaults if not specified) +- `max_h3_tags` (optional): Maximum H3 subheadings total (uses defaults if not specified) + +## Tier Defaults + +If tier parameters are not specified, these defaults are used: + +### tier1 +- `min_word_count`: 2000 +- `max_word_count`: 2500 +- `min_h2_tags`: 3 +- `max_h2_tags`: 5 +- `min_h3_tags`: 5 +- `max_h3_tags`: 10 + +### tier2 +- `min_word_count`: 1500 +- `max_word_count`: 2000 +- `min_h2_tags`: 2 +- `max_h2_tags`: 4 +- `min_h3_tags`: 3 +- `max_h3_tags`: 8 + +### tier3 +- `min_word_count`: 1000 +- `max_word_count`: 1500 +- `min_h2_tags`: 2 +- `max_h2_tags`: 3 +- `min_h3_tags`: 2 +- `max_h3_tags`: 6 + +## Examples + +### Simple: Single Tier with Defaults +```json +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 5 + } + } + } + ] +} +``` + +### Custom Word Counts +```json +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 3, + "min_word_count": 2500, + "max_word_count": 3000 + } + } + } + ] +} +``` + +### Multi-Tier +```json +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 5 + }, + "tier2": { + "count": 10 + }, + "tier3": { + "count": 15 + } + } + } + ] +} +``` + +### Multiple Projects +```json +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 5 + } + } + }, + { + "project_id": 2, + "tiers": { + "tier1": { + "count": 3 + }, + "tier2": { + "count": 8 + } + } + } + ] +} +``` + +## Usage + +Run batch generation with: + +```bash +python main.py generate-batch --job-file jobs/example_tier1_batch.json --username youruser --password yourpass +``` + +### Options +- `--job-file, -j`: Path to job JSON file (required) +- `--username, -u`: Username for authentication +- `--password, -p`: Password for authentication +- `--debug`: Save AI responses to debug_output/ +- `--continue-on-error`: Continue processing if article generation fails +- `--model, -m`: AI model to use (default: gpt-4o-mini) + +### Debug Mode + +When using `--debug`, AI responses are saved to `debug_output/`: +- `title_project{id}_tier{tier}_article{n}_{timestamp}.txt` +- `outline_project{id}_tier{tier}_article{n}_{timestamp}.json` +- `content_project{id}_tier{tier}_article{n}_{timestamp}.html` +- `augmented_project{id}_tier{tier}_article{n}_{timestamp}.html` (if augmented) + diff --git a/jobs/example_multi_tier_batch.json b/jobs/example_multi_tier_batch.json new file mode 100644 index 0000000..84ae16a --- /dev/null +++ b/jobs/example_multi_tier_batch.json @@ -0,0 +1,30 @@ +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 5, + "min_word_count": 2200, + "max_word_count": 2600 + }, + "tier2": { + "count": 10 + }, + "tier3": { + "count": 15, + "max_h2_tags": 4 + } + } + }, + { + "project_id": 2, + "tiers": { + "tier1": { + "count": 3 + } + } + } + ] +} + diff --git a/jobs/example_tier1_batch.json b/jobs/example_tier1_batch.json new file mode 100644 index 0000000..810fb80 --- /dev/null +++ b/jobs/example_tier1_batch.json @@ -0,0 +1,13 @@ +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 5 + } + } + } + ] +} + diff --git a/jobs/test_augmentation.json b/jobs/test_augmentation.json new file mode 100644 index 0000000..9468b97 --- /dev/null +++ b/jobs/test_augmentation.json @@ -0,0 +1,19 @@ +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 1, + "min_word_count": 2000, + "max_word_count": 2500, + "min_h2_tags": 3, + "max_h2_tags": 5, + "min_h3_tags": 5, + "max_h3_tags": 10 + } + } + } + ] +} + diff --git a/jobs/test_small.json b/jobs/test_small.json new file mode 100644 index 0000000..d496fe6 --- /dev/null +++ b/jobs/test_small.json @@ -0,0 +1,19 @@ +{ + "jobs": [ + { + "project_id": 1, + "tiers": { + "tier1": { + "count": 1, + "min_word_count": 500, + "max_word_count": 800, + "min_h2_tags": 2, + "max_h2_tags": 3, + "min_h3_tags": 3, + "max_h3_tags": 6 + } + } + } + ] +} + diff --git a/scripts/add_admin_direct.py b/scripts/add_admin_direct.py new file mode 100644 index 0000000..9656e54 --- /dev/null +++ b/scripts/add_admin_direct.py @@ -0,0 +1,27 @@ +import sys +from pathlib import Path +project_root = Path(__file__).parent.parent +sys.path.insert(0, str(project_root)) + +from src.database.session import db_manager +from src.database.repositories import UserRepository +from src.auth.service import AuthService + +db_manager.initialize() +session = db_manager.get_session() + +try: + user_repo = UserRepository(session) + auth_service = AuthService(user_repo) + + user = auth_service.create_user_with_hashed_password( + username="admin", + password="admin1234", + role="Admin" + ) + + print(f"Admin user created: {user.username}") +finally: + session.close() + db_manager.close() + diff --git a/src/cli/commands.py b/src/cli/commands.py index 373a2b9..94112a2 100644 --- a/src/cli/commands.py +++ b/src/cli/commands.py @@ -16,6 +16,11 @@ from src.deployment.bunnynet import ( BunnyNetResourceConflictError ) from src.ingestion.parser import CORAParser, CORAParseError +from src.generation.ai_client import AIClient, PromptManager +from src.generation.service import ContentGenerator +from src.generation.batch_processor import BatchProcessor +from src.database.repositories import GeneratedContentRepository +import os def authenticate_admin(username: str, password: str) -> Optional[User]: @@ -871,5 +876,84 @@ def list_projects(username: Optional[str], password: Optional[str]): raise click.Abort() +@app.command("generate-batch") +@click.option('--job-file', '-j', required=True, type=click.Path(exists=True), + help='Path to job JSON file') +@click.option('--username', '-u', help='Username for authentication') +@click.option('--password', '-p', help='Password for authentication') +@click.option('--debug', is_flag=True, help='Save AI responses to debug_output/') +@click.option('--continue-on-error', is_flag=True, + help='Continue processing if article generation fails') +@click.option('--model', '-m', default='gpt-4o-mini', + help='AI model to use (gpt-4o-mini, claude-sonnet-4.5)') +def generate_batch( + job_file: str, + username: Optional[str], + password: Optional[str], + debug: bool, + continue_on_error: bool, + model: str +): + """Generate content batch from job file""" + try: + if not username or not password: + username, password = prompt_admin_credentials() + + session = db_manager.get_session() + try: + user_repo = UserRepository(session) + auth_service = AuthService(user_repo) + + user = auth_service.authenticate_user(username, password) + if not user: + click.echo("Error: Authentication failed", err=True) + raise click.Abort() + + click.echo(f"Authenticated as: {user.username} ({user.role})") + + api_key = os.getenv("OPENROUTER_API_KEY") + if not api_key: + click.echo("Error: OPENROUTER_API_KEY not found in environment", err=True) + click.echo("Please set OPENROUTER_API_KEY in your .env file", err=True) + raise click.Abort() + + click.echo(f"Initializing AI client with model: {model}") + ai_client = AIClient(api_key=api_key, model=model) + prompt_manager = PromptManager() + + project_repo = ProjectRepository(session) + content_repo = GeneratedContentRepository(session) + + content_generator = ContentGenerator( + ai_client=ai_client, + prompt_manager=prompt_manager, + project_repo=project_repo, + content_repo=content_repo + ) + + batch_processor = BatchProcessor( + content_generator=content_generator, + content_repo=content_repo, + project_repo=project_repo + ) + + click.echo(f"\nProcessing job file: {job_file}") + if debug: + click.echo("Debug mode: AI responses will be saved to debug_output/\n") + + batch_processor.process_job( + job_file_path=job_file, + debug=debug, + continue_on_error=continue_on_error + ) + + finally: + session.close() + + except Exception as e: + click.echo(f"Error processing batch: {e}", err=True) + raise click.Abort() + + if __name__ == "__main__": app() diff --git a/src/database/models.py b/src/database/models.py index f536df3..1413b41 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -3,7 +3,7 @@ SQLAlchemy database models """ from datetime import datetime, timezone -from typing import Literal, Optional +from typing import Optional from sqlalchemy import String, Integer, DateTime, Float, ForeignKey, JSON, Text from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column @@ -115,4 +115,29 @@ class Project(Base): ) def __repr__(self) -> str: - return f"tags). You can add new paragraphs, expand existing ones, or add new subsections as needed. Do NOT change the existing headings unless necessary." +} + diff --git a/src/generation/prompts/content_generation.json b/src/generation/prompts/content_generation.json new file mode 100644 index 0000000..cb97125 --- /dev/null +++ b/src/generation/prompts/content_generation.json @@ -0,0 +1,5 @@ +{ + "system_message": "You are an expert content writer who creates engaging, informative, and SEO-optimized articles that provide real value to readers while incorporating relevant keywords naturally.", + "user_prompt": "Write a complete article based on:\nTitle: {title}\nOutline: {outline}\nKeyword: {keyword}\n\nEntities to include naturally: {entities}\nRelated searches to address: {related_searches}\n\nTarget word count range: {min_word_count} to {max_word_count} words\n\nReturn as an HTML fragment with
tags. Do NOT include , ,
, or tags. Start directly with the firsttags + """ + project = self.project_repo.get_by_id(project_id) + if not project: + raise ValueError(f"Project {project_id} not found") + + entities_str = ", ".join(project.entities or []) + related_str = ", ".join(project.related_searches or []) + outline_str = json.dumps(outline, indent=2) + + system_msg, user_prompt = self.prompt_manager.format_prompt( + "content_generation", + title=title, + outline=outline_str, + keyword=project.main_keyword, + entities=entities_str, + related_searches=related_str, + min_word_count=min_word_count, + max_word_count=max_word_count + ) + + content = self.ai_client.generate_completion( + prompt=user_prompt, + system_message=system_msg, + max_tokens=8000, + temperature=0.7 + ) + + content = content.strip() + + if debug: + self._save_debug_output( + project_id, "content", content, "html" + ) + + return content + + def validate_word_count(self, content: str, min_words: int, max_words: int) -> Tuple[bool, int]: + """ + Validate content word count + + Args: + content: HTML content string + min_words: Minimum word count + max_words: Maximum word count + + Returns: + Tuple of (is_valid, actual_count) + """ + word_count = self.count_words(content) + is_valid = min_words <= word_count <= max_words + return is_valid, word_count + + def count_words(self, html_content: str) -> int: + """ + Count words in HTML content + + Args: + html_content: HTML string + + Returns: + Number of words + """ + text = re.sub(r'<[^>]+>', '', html_content) + text = unescape(text) + words = text.split() + return len(words) + + def augment_content( + self, + content: str, + target_word_count: int, + debug: bool = False, + project_id: Optional[int] = None + ) -> str: + """ + Expand article content to meet minimum word count + + Args: + content: Current HTML content + target_word_count: Target word count + debug: If True, save response to debug_output/ + project_id: Optional project ID for debug output + + Returns: + Expanded HTML content + """ + system_msg, user_prompt = self.prompt_manager.format_prompt( + "content_augmentation", + content=content, + target_word_count=target_word_count + ) + + augmented = self.ai_client.generate_completion( + prompt=user_prompt, + system_message=system_msg, + max_tokens=8000, + temperature=0.7 + ) + + augmented = augmented.strip() + + if debug and project_id: + self._save_debug_output( + project_id, "augmented", augmented, "html" + ) + + return augmented + + def _save_debug_output( + self, + project_id: int, + stage: str, + content: str, + extension: str, + tier: Optional[str] = None, + article_num: Optional[int] = None + ): + """Save debug output to file""" + debug_dir = Path("debug_output") + debug_dir.mkdir(exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + + tier_part = f"_tier{tier}" if tier else "" + article_part = f"_article{article_num}" if article_num else "" + + filename = f"{stage}_project{project_id}{tier_part}{article_part}_{timestamp}.{extension}" + filepath = debug_dir / filename + + with open(filepath, 'w', encoding='utf-8') as f: + f.write(content) diff --git a/tests/integration/test_generate_batch.py b/tests/integration/test_generate_batch.py new file mode 100644 index 0000000..4fb7c9f --- /dev/null +++ b/tests/integration/test_generate_batch.py @@ -0,0 +1,52 @@ +""" +Integration test for batch generation (stub) +""" + +import pytest +from unittest.mock import Mock, MagicMock +from src.generation.batch_processor import BatchProcessor +from src.generation.service import ContentGenerator + + +def test_batch_processor_initialization(): + """Test BatchProcessor can be initialized""" + mock_generator = Mock(spec=ContentGenerator) + mock_content_repo = Mock() + mock_project_repo = Mock() + + processor = BatchProcessor( + content_generator=mock_generator, + content_repo=mock_content_repo, + project_repo=mock_project_repo + ) + + assert processor is not None + assert processor.stats["total_jobs"] == 0 + assert processor.stats["processed_jobs"] == 0 + + +def test_batch_processor_stats_initialization(): + """Test BatchProcessor initializes stats correctly""" + mock_generator = Mock(spec=ContentGenerator) + mock_content_repo = Mock() + mock_project_repo = Mock() + + processor = BatchProcessor( + content_generator=mock_generator, + content_repo=mock_content_repo, + project_repo=mock_project_repo + ) + + expected_keys = [ + "total_jobs", + "processed_jobs", + "total_articles", + "generated_articles", + "augmented_articles", + "failed_articles" + ] + + for key in expected_keys: + assert key in processor.stats + assert processor.stats[key] == 0 + diff --git a/tests/unit/test_content_generator.py b/tests/unit/test_content_generator.py new file mode 100644 index 0000000..a4ace5e --- /dev/null +++ b/tests/unit/test_content_generator.py @@ -0,0 +1,95 @@ +""" +Unit tests for ContentGenerator service +""" + +import pytest +from src.generation.service import ContentGenerator + + +def test_count_words_simple(): + """Test word count on simple text""" + generator = ContentGenerator(None, None, None, None) + + html = "
This is a test with five words
" + count = generator.count_words(html) + + assert count == 7 + + +def test_count_words_with_headings(): + """Test word count with HTML headings""" + generator = ContentGenerator(None, None, None, None) + + html = """ +This is a paragraph with some words.
+Another paragraph here.
+ """ + + count = generator.count_words(html) + + assert count > 10 + + +def test_count_words_strips_html_tags(): + """Test that HTML tags are stripped before counting""" + generator = ContentGenerator(None, None, None, None) + + html = "Hello world this is a test
" + count = generator.count_words(html) + + assert count == 6 + + +def test_validate_word_count_within_range(): + """Test validation when word count is within range""" + generator = ContentGenerator(None, None, None, None) + + content = "" + " ".join(["word"] * 100) + "
" + is_valid, count = generator.validate_word_count(content, 50, 150) + + assert is_valid is True + assert count == 100 + + +def test_validate_word_count_below_minimum(): + """Test validation when word count is below minimum""" + generator = ContentGenerator(None, None, None, None) + + content = "" + " ".join(["word"] * 30) + "
" + is_valid, count = generator.validate_word_count(content, 50, 150) + + assert is_valid is False + assert count == 30 + + +def test_validate_word_count_above_maximum(): + """Test validation when word count is above maximum""" + generator = ContentGenerator(None, None, None, None) + + content = "" + " ".join(["word"] * 200) + "
" + is_valid, count = generator.validate_word_count(content, 50, 150) + + assert is_valid is False + assert count == 200 + + +def test_count_words_empty_content(): + """Test word count on empty content""" + generator = ContentGenerator(None, None, None, None) + + count = generator.count_words("") + + assert count == 0 + + +def test_count_words_only_tags(): + """Test word count on content with only HTML tags""" + generator = ContentGenerator(None, None, None, None) + + html = "