""" Story discovery and ebook build pipeline. Scans the vault for metadata.md files (with YAML frontmatter), discovers chapter files, merges them into a single markdown document, exports a DOCX via pandoc, then converts that DOCX to EPUB and MOBI. """ import logging import os import re import subprocess import tempfile import unicodedata from pathlib import Path from typing import Optional import yaml logger = logging.getLogger(__name__) METADATA_FILENAME = "metadata.md" DEFAULT_OUTPUT_FILE_MODE = 0o664 EPUB_SUBDIR = "epub" MOBI_SUBDIR = "mobi" REFERENCE_DOC_PATH = Path("/app/reference.docx") DEFAULT_OUTPUT_UID = 99 DEFAULT_OUTPUT_GID = 100 def parse_frontmatter(text: str) -> dict: """Extract YAML frontmatter from a markdown file.""" text = text.strip() if not text.startswith("---"): return {} end = text.find("---", 3) if end == -1: return {} yaml_block = text[3:end].strip() try: return yaml.safe_load(yaml_block) or {} except yaml.YAMLError as e: logger.error(f"Failed to parse YAML frontmatter: {e}") return {} def write_frontmatter(metadata: dict, original_text: str) -> str: """Replace the YAML frontmatter in a markdown file, preserving body content.""" body = "" original_text = original_text.strip() if original_text.startswith("---"): end = original_text.find("---", 3) if end != -1: body = original_text[end + 3:].strip() yaml_str = yaml.dump(metadata, default_flow_style=False, allow_unicode=True).strip() result = f"---\n{yaml_str}\n---" if body: result += f"\n\n{body}" return result + "\n" def natural_sort_key(path: Path) -> tuple: """Sort chapter files by chapter number when possible.""" name = path.stem match = re.search(r"[Cc]hapter\s+(\d+)", name) if match: return (0, int(match.group(1)), name) match = re.match(r"(\d+)", name) if match: return (0, int(match.group(1)), name) return (1, 0, name) def discover_stories(vault_path: Path, creative_folder: str = "Creative") -> list[dict]: """Recursively scan for metadata.md files under the creative folder.""" creative_path = vault_path / creative_folder if not creative_path.exists(): logger.warning(f"Creative folder not found: {creative_path}") return [] stories = [] for metadata_file in creative_path.rglob(METADATA_FILENAME): story_dir = metadata_file.parent try: raw_text = metadata_file.read_text(encoding="utf-8") metadata = parse_frontmatter(raw_text) except Exception as e: logger.error(f"Failed to parse {metadata_file}: {e}") continue if not metadata: logger.warning(f"No valid frontmatter in {metadata_file}, skipping") continue chapters = sorted( [f for f in story_dir.iterdir() if f.suffix == ".md" and f.name != METADATA_FILENAME], key=natural_sort_key, ) stories.append( { "path": story_dir, "metadata_file": metadata_file, "metadata": metadata, "metadata_raw": raw_text, "chapters": chapters, } ) logger.info( f"Discovered story: {metadata.get('title', story_dir.name)} " f"({len(chapters)} chapters)" ) return stories def merge_chapters(chapters: list[Path], add_page_breaks: bool = True) -> str: """Merge chapter markdown files into a single document.""" merged_parts = [] for i, chapter_file in enumerate(chapters): try: content = chapter_file.read_text(encoding="utf-8").strip() except Exception as e: logger.error(f"Failed to read {chapter_file}: {e}") continue if i > 0 and add_page_breaks: # Use Pandoc's explicit page break marker so DOCX gets a real break. merged_parts.append("\n\n\\newpage\n\n") merged_parts.append(content) return "\n\n".join(merged_parts) def normalize_text_for_docx(text: str) -> str: """Normalize markdown text before DOCX export. NFC is a conservative choice that can reduce weirdness around combining characters without stripping the visual effect from decorated text. """ return unicodedata.normalize("NFC", text) def sanitize_title(title: str) -> str: """Convert a story title into a safe base filename.""" return re.sub(r"[^\w\s-]", "", title).strip() def get_cover_path(story: dict) -> Optional[Path]: """Resolve a configured or auto-detected cover image for the story.""" metadata = story["metadata"] cover = metadata.get("cover") if cover: cover_path = story["path"] / cover if cover_path.exists(): return cover_path logger.warning(f"Cover image specified but not found: {cover_path}") return None for ext in ["jpg", "jpeg", "png"]: candidate = story["path"] / f"cover.{ext}" if candidate.exists(): return candidate return None def set_output_permissions(*paths: Path): """Set predictable readable permissions on generated output files.""" for path in paths: os.chmod(path, DEFAULT_OUTPUT_FILE_MODE) def set_output_ownership(*paths: Path): """Set predictable ownership on generated output paths.""" for path in paths: os.chown(path, DEFAULT_OUTPUT_UID, DEFAULT_OUTPUT_GID) def run_command(cmd: list[str], title: str, output_label: str) -> bool: """Run an external command and log any failure details.""" logger.debug(f"Running: {' '.join(cmd)}") result = subprocess.run( cmd, capture_output=True, text=True, timeout=240, ) if result.returncode != 0: logger.error(f"{output_label} build failed for '{title}':\n{result.stderr}") return False if result.stderr: logger.warning(f"{output_label} warnings for '{title}':\n{result.stderr}") return True def build_outputs( story: dict, output_dir: Path, lua_filter: Optional[Path] = None, pagebreak_filter: Optional[Path] = None, css_file: Optional[Path] = None, ) -> Optional[dict[str, Path]]: """Build DOCX, EPUB, and MOBI outputs for a story.""" metadata = story["metadata"] title = metadata.get("title", story["path"].name) author = metadata.get("author", "Unknown Author") language = metadata.get("language", "en") safe_title = sanitize_title(title) epub_dir = output_dir / EPUB_SUBDIR mobi_dir = output_dir / MOBI_SUBDIR docx_file = output_dir / f"{safe_title}.docx" epub_file = epub_dir / f"{safe_title}.epub" mobi_file = mobi_dir / f"{safe_title}.mobi" output_dir.mkdir(parents=True, exist_ok=True) epub_dir.mkdir(parents=True, exist_ok=True) mobi_dir.mkdir(parents=True, exist_ok=True) set_output_ownership(output_dir, epub_dir, mobi_dir) if not story["chapters"]: logger.warning(f"No chapters found for '{title}', skipping") return None logger.info(f"Building book files: '{title}' ({len(story['chapters'])} chapters)") merged_content = normalize_text_for_docx(merge_chapters(story["chapters"])) cover_path = get_cover_path(story) with tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False, encoding="utf-8") as tmp: tmp.write(merged_content) tmp_path = tmp.name try: docx_cmd = [ "pandoc", tmp_path, "-o", str(docx_file), "--from", "markdown+raw_html+raw_tex", "--to", "docx", "--metadata", f"title={title}", "--metadata", f"author={author}", "--metadata", f"lang={language}", "--wrap=none", ] if REFERENCE_DOC_PATH.exists(): docx_cmd.extend(["--reference-doc", str(REFERENCE_DOC_PATH)]) if lua_filter and lua_filter.exists(): docx_cmd.extend(["--lua-filter", str(lua_filter)]) if pagebreak_filter and pagebreak_filter.exists(): docx_cmd.extend(["--lua-filter", str(pagebreak_filter)]) if not run_command(docx_cmd, title, "DOCX"): return None logger.info(f"DOCX created: {docx_file}") base_convert_cmd = [ "--title", title, "--authors", author, "--language", language, "--chapter", "//h:h1|//h:h2", "--level1-toc", "//h:h1", "--level2-toc", "//h:h2", "--max-toc-links", "500", ] epub_cmd = ["ebook-convert", str(docx_file), str(epub_file), *base_convert_cmd] mobi_cmd = ["ebook-convert", str(docx_file), str(mobi_file), *base_convert_cmd] if cover_path: logger.info(f"Using cover image: {cover_path.name}") epub_cmd.extend(["--cover", str(cover_path)]) mobi_cmd.extend(["--cover", str(cover_path)]) if css_file and css_file.exists(): epub_cmd.extend(["--extra-css", str(css_file)]) mobi_cmd.extend(["--extra-css", str(css_file)]) if not run_command(epub_cmd, title, "EPUB"): return None logger.info(f"EPUB created: {epub_file}") if not run_command(mobi_cmd, title, "MOBI"): return None logger.info(f"MOBI created: {mobi_file}") set_output_permissions(docx_file, epub_file, mobi_file) set_output_ownership(docx_file, epub_file, mobi_file) return {"docx": docx_file, "epub": epub_file, "mobi": mobi_file} except subprocess.TimeoutExpired: logger.error(f"Build timed out for '{title}'") except Exception as e: logger.error(f"Build failed for '{title}': {e}") finally: Path(tmp_path).unlink(missing_ok=True) return None