From cea352b576f1fcb40abcfd3e3e2457c829fe7ad1 Mon Sep 17 00:00:00 2001 From: null3FF3KT Date: Sat, 19 Sep 2026 10:56:57 -0500 Subject: [PATCH] Add local document summarization and custom prompt mode Summarize local PDF, Word, and plain text files via a new DocumentService (PdfPig + DocumentFormat.OpenXml), and add a Custom summary mode that uses the user's own instructions as the system prompt. Update the console UI, transcript saving, and README to match. Co-Authored-By: Claude Sonnet 5 --- ConsoleRenderer.cs | 51 ++++++-- DocumentService.cs | 198 ++++++++++++++++++++++++++++++ Program.cs | 254 +++++++++++++++++++++++++++++---------- README.md | 171 ++++++++++++++++---------- SummarizerService.cs | 33 ++++- TranscriptFileService.cs | 22 +++- VideoModels.cs | 19 ++- YoutubeSummarizer.csproj | 2 + 8 files changed, 600 insertions(+), 150 deletions(-) create mode 100644 DocumentService.cs diff --git a/ConsoleRenderer.cs b/ConsoleRenderer.cs index fcab2ec..e7b19e2 100644 --- a/ConsoleRenderer.cs +++ b/ConsoleRenderer.cs @@ -51,15 +51,15 @@ public static class ConsoleRenderer // Revert the title while waiting for the next video. SetTitle("Summarize"); + const string label = "[bold]Enter a YouTube URL or a local file path[/] [grey](or 'q' to quit)[/]"; + if (!Interactive) { - AnsiConsole.Markup("[bold]Enter YouTube URL[/] [grey](or 'q' to quit)[/]: "); + AnsiConsole.Markup($"{label}: "); return Console.ReadLine()?.Trim() ?? string.Empty; } - var input = AnsiConsole.Prompt( - new TextPrompt("[bold]Enter YouTube URL[/] [grey](or 'q' to quit)[/]:") - .AllowEmpty()); + var input = AnsiConsole.Prompt(new TextPrompt($"{label}:").AllowEmpty()); return input.Trim(); } @@ -91,15 +91,36 @@ public static class ConsoleRenderer return AnsiConsole.Prompt( new SelectionPrompt() .Title("[bold]Choose summary mode:[/]") - .AddChoices(SummaryMode.Standard, SummaryMode.PersonalFilter) + .AddChoices(SummaryMode.Standard, SummaryMode.PersonalFilter, SummaryMode.Custom) .UseConverter(m => m switch { SummaryMode.Standard => "Standard – detailed bullet-point summary", SummaryMode.PersonalFilter => "Personal Filter – relevance verdict (ACT / MONITOR / IGNORE)", + SummaryMode.Custom => "Custom – write your own instructions", _ => m.ToString() })); } + /// + /// Reads multi-line custom summarization instructions from the user, + /// terminated by a blank line. Spectre has no built-in multi-line text + /// prompt, so this reads raw lines directly (still styled with a Spectre + /// header) and joins them with newlines. + /// + public static string PromptCustomInstructions() + { + AnsiConsole.MarkupLine("[bold]Enter your custom instructions[/] [grey](finish with an empty line):[/]"); + + var lines = new List(); + string? line; + while (!string.IsNullOrEmpty(line = Console.ReadLine())) + { + lines.Add(line); + } + + return string.Join('\n', lines).Trim(); + } + /// Prints a dim "working" indicator before an async step. public static void PrintWorking(string message) { @@ -115,16 +136,27 @@ public static class ConsoleRenderer { AnsiConsole.WriteLine(); + var isDocument = summary.TranscriptSource == TranscriptSource.LocalDocument; + // ── Metadata header ────────────────────────────────────────────────── var grid = new Grid(); grid.AddColumn(new GridColumn().PadRight(2)); grid.AddColumn(); grid.AddRow("[bold green]Title[/]", $"[bold]{Markup.Escape(summary.Metadata.Title)}[/]"); - grid.AddRow("[grey]Channel[/]", Markup.Escape(summary.Metadata.ChannelTitle)); - grid.AddRow("[grey]Published[/]", Markup.Escape(summary.Metadata.PublishedAt.ToString("MMMM d, yyyy"))); - grid.AddRow("[grey]Duration[/]", Markup.Escape(summary.Metadata.FormattedDuration)); - grid.AddRow("[grey]URL[/]", $"https://youtu.be/{Markup.Escape(summary.Metadata.VideoId)}"); + grid.AddRow(isDocument ? "[grey]Type[/]" : "[grey]Channel[/]", Markup.Escape(summary.Metadata.ChannelTitle)); + grid.AddRow( + isDocument ? "[grey]Modified[/]" : "[grey]Published[/]", + Markup.Escape(summary.Metadata.PublishedAt.ToString("MMMM d, yyyy"))); + + if (summary.Metadata.Duration is not null) + grid.AddRow("[grey]Duration[/]", Markup.Escape(summary.Metadata.FormattedDuration)); + + grid.AddRow( + isDocument ? "[grey]Location[/]" : "[grey]URL[/]", + isDocument + ? Markup.Escape(summary.Metadata.SourcePath ?? summary.Metadata.Title) + : $"https://youtu.be/{Markup.Escape(summary.Metadata.VideoId)}"); if (showTranscriptSource) { @@ -134,6 +166,7 @@ public static class ConsoleRenderer TranscriptSource.CommunityContributed => ("✓ Community captions", "green"), TranscriptSource.AutoGenerated => ("~ Auto-generated (ASR)", "yellow"), TranscriptSource.MetadataOnly => ("✗ Metadata only", "red"), + TranscriptSource.LocalDocument => ("✓ Local document", "green"), _ => ("? Unknown", "grey") }; grid.AddRow("[grey]Transcript[/]", $"[{color}]{Markup.Escape(badge)}[/]"); diff --git a/DocumentService.cs b/DocumentService.cs new file mode 100644 index 0000000..d8b251d --- /dev/null +++ b/DocumentService.cs @@ -0,0 +1,198 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using System.Xml.Linq; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Spreadsheet; +using UglyToad.PdfPig; +using YoutubeSummarizer.Models; + +namespace YoutubeSummarizer.Services; + +/// +/// Reads a local file — text, Markdown, CSV, PDF, Word, OpenDocument text, or +/// Excel — and adapts it into the same / +/// shapes the YouTube pipeline uses, so the rest +/// of the app (summarization, rendering, saving) doesn't need to know or care +/// where the text came from. +/// +public static class DocumentService +{ + /// File extensions this service knows how to read (lowercase, with leading dot). + private static readonly HashSet SupportedExtensions = new(StringComparer.OrdinalIgnoreCase) + { + ".txt", ".md", ".csv", ".pdf", ".docx", ".odt", ".xlsx" + }; + + /// True when has an extension this service can extract text from. + public static bool IsSupportedFile(string path) => + SupportedExtensions.Contains(Path.GetExtension(path)); + + /// A human-readable label for the file type, shown in place of "Channel" for documents. + public static string FriendlyTypeName(string path) => Path.GetExtension(path).ToLowerInvariant() switch + { + ".txt" => "Plain text", + ".md" => "Markdown document", + ".csv" => "Spreadsheet (CSV)", + ".xlsx" => "Spreadsheet", + ".pdf" => "PDF document", + ".docx" => "Word document", + ".odt" => "OpenDocument text", + _ => "Document" + }; + + /// Builds synthetic video-shaped metadata describing a local file. + public static VideoMetadata BuildMetadata(string path) + { + var fullPath = Path.GetFullPath(path); + + return new VideoMetadata + { + VideoId = ShortHash(fullPath), + Title = Path.GetFileName(fullPath), + ChannelTitle = FriendlyTypeName(fullPath), + PublishedAt = File.GetLastWriteTimeUtc(fullPath), + Duration = null, + Description = null, + SourcePath = fullPath + }; + } + + /// Extracts plain text from the file and wraps it as a transcript. + public static async Task ExtractTextAsync(string path, CancellationToken ct = default) + { + var extension = Path.GetExtension(path).ToLowerInvariant(); + + var text = extension switch + { + ".txt" or ".md" or ".csv" => await File.ReadAllTextAsync(path, ct), + ".pdf" => ExtractPdfText(path), + ".docx" => ExtractDocxText(path), + ".xlsx" => ExtractXlsxText(path), + ".odt" => ExtractOdtText(path), + _ => throw new NotSupportedException($"Unsupported file type: {extension}") + }; + + return new VideoTranscript + { + VideoId = ShortHash(Path.GetFullPath(path)), + Text = text, + SourceTrack = null, + Source = TranscriptSource.LocalDocument + }; + } + + // ───────────────────────────────────────────────────────────────────────── + // Extraction helpers + // ───────────────────────────────────────────────────────────────────────── + + /// + /// Joins the words of every page of a PDF with spaces (rather than using + /// Page.Text directly, which concatenates line/column breaks with no + /// separator and jams words together, e.g. "...StatusThis week..."). + /// + private static string ExtractPdfText(string path) + { + using var document = PdfDocument.Open(path); + var pages = document.GetPages().Select(p => string.Join(" ", p.GetWords().Select(w => w.Text))); + return string.Join("\n\n", pages); + } + + /// Walks the body paragraphs of a .docx and joins their text with newlines. + private static string ExtractDocxText(string path) + { + using var doc = WordprocessingDocument.Open(path, isEditable: false); + var body = doc.MainDocumentPart?.Document?.Body; + if (body is null) return string.Empty; + + // Join per paragraph rather than Body.InnerText, which flattens every + // descendant text node with no separator and jams paragraphs together. + var paragraphs = body.Elements() + .Select(p => p.InnerText); + return string.Join("\n", paragraphs); + } + + /// + /// Serializes every worksheet of an .xlsx into plain comma-separated text, + /// one sheet header + one line per row, so the LLM can reason over tabular + /// data the same way it does over transcript text. + /// + private static string ExtractXlsxText(string path) + { + using var doc = SpreadsheetDocument.Open(path, isEditable: false); + if (doc.WorkbookPart?.Workbook is not { } workbook) return string.Empty; + var workbookPart = doc.WorkbookPart; + + var sharedStrings = workbookPart.SharedStringTablePart?.SharedStringTable? + .Elements() + .Select(s => s.InnerText) + .ToList() ?? new List(); + + var sb = new StringBuilder(); + + foreach (var sheet in workbook.Descendants()) + { + if (sheet.Id?.Value is not { } relId) continue; + if (workbookPart.GetPartById(relId) is not WorksheetPart worksheetPart) continue; + if (worksheetPart.Worksheet is not { } worksheet) continue; + + sb.AppendLine($"## Sheet: {sheet.Name}"); + + foreach (var row in worksheet.Descendants()) + { + var cells = row.Elements().Select(cell => GetCellText(cell, sharedStrings)); + sb.AppendLine(string.Join(",", cells)); + } + + sb.AppendLine(); + } + + return sb.ToString(); + } + + /// Resolves a cell's display text, following the shared-strings table when needed. + private static string GetCellText(Cell cell, List sharedStrings) + { + var value = cell.CellValue?.InnerText ?? string.Empty; + + if (cell.DataType?.Value == CellValues.SharedString && + int.TryParse(value, out var index) && + index >= 0 && index < sharedStrings.Count) + { + return sharedStrings[index]; + } + + return value; + } + + /// + /// An .odt is a zip archive; the visible text lives in content.xml as a + /// sequence of ODF text:p / text:h elements. No OpenDocument NuGet package + /// is pulled in for this — the format is simple enough to read directly, + /// the same way YouTubeService hand-parses VTT/timedtext captions. + /// + private static string ExtractOdtText(string path) + { + const string textNamespace = "urn:oasis:names:tc:opendocument:xmlns:text:1.0"; + + using var archive = ZipFile.OpenRead(path); + var contentEntry = archive.GetEntry("content.xml") + ?? throw new InvalidDataException("Not a valid .odt file (missing content.xml)."); + + using var stream = contentEntry.Open(); + var doc = XDocument.Load(stream); + + var paragraphs = doc.Descendants() + .Where(el => el.Name == XName.Get("p", textNamespace) || el.Name == XName.Get("h", textNamespace)) + .Select(el => el.Value); + + return string.Join("\n", paragraphs); + } + + /// Short, stable, filesystem-safe identifier derived from a full path (mirrors a YouTube video ID's shape). + private static string ShortHash(string fullPath) + { + var bytes = SHA1.HashData(Encoding.UTF8.GetBytes(fullPath)); + return Convert.ToHexString(bytes)[..8].ToLowerInvariant(); + } +} diff --git a/Program.cs b/Program.cs index 2aa46c0..9c8e425 100644 --- a/Program.cs +++ b/Program.cs @@ -70,22 +70,45 @@ while (!cts.Token.IsCancellationRequested) if (string.IsNullOrWhiteSpace(input)) continue; if (input.Equals("q", StringComparison.OrdinalIgnoreCase)) break; + // Auto-detect: an existing local file wins over YouTube URL parsing — + // a real path on disk is an unambiguous signal, whereas a bare 11-char + // YouTube ID could theoretically collide with a short filename. + var candidatePath = NormalizeCandidatePath(input); + + if (File.Exists(candidatePath)) + { + if (!DocumentService.IsSupportedFile(candidatePath)) + { + ConsoleRenderer.PrintError($"Unsupported file type: {Path.GetExtension(candidatePath)}"); + ConsoleRenderer.PrintWarning("Supported: .txt, .md, .csv, .pdf, .docx, .odt, .xlsx"); + continue; + } + + var saveTranscript = ConsoleRenderer.PromptSaveTranscript(); + var (summaryMode, customPrompt) = PromptModeAndCustomPrompt(); + + await ProcessDocumentAsync( + candidatePath, serviceProvider, appSettings.Summarizer, + saveTranscript, summaryMode, customPrompt, cts.Token); + continue; + } + // Parse the video ID from the URL var videoId = YouTubeService.ExtractVideoId(input); if (videoId is null) { - ConsoleRenderer.PrintError("Could not extract a valid YouTube video ID from that URL."); - ConsoleRenderer.PrintWarning("Accepted formats: watch?v=..., youtu.be/..., /shorts/..., /embed/..."); + ConsoleRenderer.PrintError("Could not extract a YouTube video ID, and no such file exists."); + ConsoleRenderer.PrintWarning( + "Accepted: watch?v=..., youtu.be/..., /shorts/..., /embed/..., or a path to an existing file."); continue; } - // Ask whether to save transcript to file before processing - var saveTranscript = ConsoleRenderer.PromptSaveTranscript(); + var saveVideoTranscript = ConsoleRenderer.PromptSaveTranscript(); + var (videoSummaryMode, videoCustomPrompt) = PromptModeAndCustomPrompt(); - // Choose summary mode - var summaryMode = ConsoleRenderer.PromptSummaryMode(); - - await ProcessVideoAsync(videoId, serviceProvider, appSettings.Summarizer, saveTranscript, summaryMode, cts.Token); + await ProcessVideoAsync( + videoId, serviceProvider, appSettings.Summarizer, + saveVideoTranscript, videoSummaryMode, videoCustomPrompt, cts.Token); } AnsiConsole.MarkupLine("[grey]Goodbye![/]"); @@ -95,11 +118,8 @@ AnsiConsole.MarkupLine("[grey]Goodbye![/]"); // ═════════════════════════════════════════════════════════════════════════════ /// -/// Orchestrates the full pipeline for a single video: -/// 1. Fetch metadata (YouTube Data API) -/// 2. Fetch transcript (caption track or timedtext fallback) -/// 3. Summarize (LLM Chat Completions) -/// 4. Display (ConsoleRenderer) +/// Fetches a YouTube video's metadata + transcript, then hands off to the +/// shared summarize/save/display pipeline. /// static async Task ProcessVideoAsync( string videoId, @@ -107,13 +127,12 @@ static async Task ProcessVideoAsync( SummarizerSettings summarizerSettings, bool saveTranscript, SummaryMode summaryMode, + string? customPrompt, CancellationToken ct) { try { - // Resolve scoped services var youtubeService = sp.GetRequiredService(); - var summarizerService = sp.GetRequiredService(); // ── Step 1: Metadata ────────────────────────────────────────────── ConsoleRenderer.PrintWorking("Fetching video metadata"); @@ -132,56 +151,9 @@ static async Task ProcessVideoAsync( ConsoleRenderer.PrintWorking("Fetching transcript"); var transcript = await youtubeService.GetTranscriptAsync(metadata, ct); - // Optionally show raw transcript for debugging / inspection - if (summarizerSettings.ShowTranscript) - { - AnsiConsole.WriteLine(); - AnsiConsole.Write(new Rule("RAW TRANSCRIPT").RuleStyle("grey")); - AnsiConsole.WriteLine(transcript.Text); - AnsiConsole.Write(new Rule("END TRANSCRIPT").RuleStyle("grey")); - AnsiConsole.WriteLine(); - } - - AnsiConsole.MarkupLine( - $" [grey]Transcript:[/] {transcript.Source} | {transcript.WordCount:N0} words"); - - // ── Step 2.5: Save transcript to file (if requested) ───────────── - // (moved after summarization so we can include the summary) - - // ── Step 3: Summarize ───────────────────────────────────────────── - // Always run the standard summary (used for file saving). - ConsoleRenderer.PrintWorking("Summarizing with LLM"); - var standardSummary = await summarizerService.SummarizeAsync( - metadata, transcript, SummaryMode.Standard, ct); - - // If the user chose Personal Filter, run a second pass for display. - VideoSummary displaySummary; - if (summaryMode == SummaryMode.PersonalFilter) - { - ConsoleRenderer.PrintWorking("Applying Personal Information Filter"); - displaySummary = await summarizerService.SummarizeAsync( - metadata, transcript, SummaryMode.PersonalFilter, ct); - } - else - { - displaySummary = standardSummary; - } - - // ── Step 3.5: Save transcript + standard summary to file ───────── - if (saveTranscript) - { - var transcriptsDir = Path.Combine( - Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), - "Downloads", "transcripts"); - ConsoleRenderer.PrintWorking("Saving transcript to file"); - var savedPath = await TranscriptFileService.SaveAsync( - metadata, transcript, summaryText: standardSummary.SummaryText, - outputDirectory: transcriptsDir, ct: ct); - ConsoleRenderer.PrintFileSaved(savedPath); - } - - // ── Step 4: Display ─────────────────────────────────────────────── - ConsoleRenderer.PrintSummary(displaySummary, showTranscriptSource: true); + await RunPipelineAsync( + metadata, transcript, sp, summarizerSettings, + saveTranscript, summaryMode, customPrompt, ct); } catch (OperationCanceledException) { @@ -198,6 +170,158 @@ static async Task ProcessVideoAsync( } } +/// +/// Reads a local document's text, wraps it as video-shaped metadata/transcript +/// (see ), then hands off to the shared +/// summarize/save/display pipeline. +/// +static async Task ProcessDocumentAsync( + string path, + IServiceProvider sp, + SummarizerSettings summarizerSettings, + bool saveTranscript, + SummaryMode summaryMode, + string? customPrompt, + CancellationToken ct) +{ + try + { + var metadata = DocumentService.BuildMetadata(path); + + ConsoleRenderer.SetTitle($"Summarize {metadata.Title}"); + AnsiConsole.MarkupLine($" [bold]{Markup.Escape(metadata.Title)}[/]"); + + ConsoleRenderer.PrintWorking("Reading document"); + var transcript = await DocumentService.ExtractTextAsync(path, ct); + + await RunPipelineAsync( + metadata, transcript, sp, summarizerSettings, + saveTranscript, summaryMode, customPrompt, ct); + } + catch (OperationCanceledException) + { + // User pressed Ctrl+C — nothing to report, the loop will exit + } + catch (Exception ex) + { + ConsoleRenderer.PrintError(ex.Message); + AnsiConsole.WriteException(ex, + ExceptionFormats.ShortenPaths | ExceptionFormats.ShortenTypes); + } +} + +/// +/// Shared tail of the pipeline once metadata + transcript text are in hand, +/// regardless of whether they came from YouTube or a local document: +/// 1. Summarize (LLM Chat Completions) +/// 2. Save transcript + summary to file (if requested) +/// 3. Display (ConsoleRenderer) +/// +static async Task RunPipelineAsync( + VideoMetadata metadata, + VideoTranscript transcript, + IServiceProvider sp, + SummarizerSettings summarizerSettings, + bool saveTranscript, + SummaryMode summaryMode, + string? customPrompt, + CancellationToken ct) +{ + var summarizerService = sp.GetRequiredService(); + + // Optionally show raw transcript for debugging / inspection + if (summarizerSettings.ShowTranscript) + { + AnsiConsole.WriteLine(); + AnsiConsole.Write(new Rule("RAW TRANSCRIPT").RuleStyle("grey")); + AnsiConsole.WriteLine(transcript.Text); + AnsiConsole.Write(new Rule("END TRANSCRIPT").RuleStyle("grey")); + AnsiConsole.WriteLine(); + } + + AnsiConsole.MarkupLine( + $" [grey]Transcript:[/] {transcript.Source} | {transcript.WordCount:N0} words"); + + // Always run the standard summary (used for file saving). + ConsoleRenderer.PrintWorking("Summarizing with LLM"); + var standardSummary = await summarizerService.SummarizeAsync( + metadata, transcript, mode: SummaryMode.Standard, ct: ct); + + // If the user chose a different mode, run a second pass for display. + VideoSummary displaySummary; + if (summaryMode != SummaryMode.Standard) + { + ConsoleRenderer.PrintWorking(summaryMode switch + { + SummaryMode.PersonalFilter => "Applying Personal Information Filter", + SummaryMode.Custom => "Applying custom instructions", + _ => "Summarizing" + }); + displaySummary = await summarizerService.SummarizeAsync( + metadata, transcript, mode: summaryMode, customPrompt: customPrompt, ct: ct); + } + else + { + displaySummary = standardSummary; + } + + // Save transcript + standard summary to file + if (saveTranscript) + { + var transcriptsDir = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + "Downloads", "transcripts"); + ConsoleRenderer.PrintWorking("Saving transcript to file"); + var savedPath = await TranscriptFileService.SaveAsync( + metadata, transcript, summaryText: standardSummary.SummaryText, + outputDirectory: transcriptsDir, ct: ct); + ConsoleRenderer.PrintFileSaved(savedPath); + } + + // Display + ConsoleRenderer.PrintSummary(displaySummary, showTranscriptSource: true); +} + +/// +/// Prompts for summary mode, and — only when the user picks Custom — for the +/// instructions themselves, re-prompting until non-empty. +/// +static (SummaryMode Mode, string? CustomPrompt) PromptModeAndCustomPrompt() +{ + var mode = ConsoleRenderer.PromptSummaryMode(); + if (mode != SummaryMode.Custom) return (mode, null); + + string? customPrompt; + do + { + customPrompt = ConsoleRenderer.PromptCustomInstructions(); + if (string.IsNullOrWhiteSpace(customPrompt)) + ConsoleRenderer.PrintWarning("Custom instructions can't be empty — try again."); + } while (string.IsNullOrWhiteSpace(customPrompt)); + + return (mode, customPrompt); +} + +/// +/// Strips quotes some terminals add around a drag-and-dropped path and +/// expands a leading "~" to the user's home directory. +/// +static string NormalizeCandidatePath(string input) +{ + var trimmed = input.Trim(); + + if (trimmed.Length >= 2 && trimmed[0] == '"' && trimmed[^1] == '"') + trimmed = trimmed[1..^1]; + + if (trimmed == "~" || trimmed.StartsWith("~/")) + { + var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + trimmed = trimmed == "~" ? home : Path.Combine(home, trimmed[2..]); + } + + return trimmed; +} + // ═════════════════════════════════════════════════════════════════════════════ // Configuration validation // ═════════════════════════════════════════════════════════════════════════════ diff --git a/README.md b/README.md index 07014b5..8073357 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,100 @@ -# YouTube Video Summarizer +# YouTube & Document Summarizer -A .NET 10 console application that fetches YouTube video transcripts and produces structured summaries using an LLM (Ollama or OpenAI). +A .NET 10 console app that summarizes YouTube videos *and* local documents — +text, Markdown, CSV, PDF, Word, OpenDocument, and Excel — using an LLM +(OpenAI or a local Ollama model). One prompt accepts either a YouTube URL or +a file path; everything else (summarize, save, display) runs through the +same pipeline. --- ## Prerequisites - [.NET 10 SDK](https://dotnet.microsoft.com/download) -- A **YouTube Data API v3** key → [Google Cloud Console](https://console.cloud.google.com) -- **Local Ollama** (Recommended) or an **OpenAI API key**. +- [**yt-dlp**](https://github.com/yt-dlp/yt-dlp) on your `PATH` — used to fetch + YouTube metadata and captions. No YouTube API key needed. + + ```bash + pip install yt-dlp + ``` + +- **Local Ollama** (recommended, free) or an **OpenAI API key** --- ## Setup ```bash -# 1. Clone / copy the project -cd YoutubeSummarizer +cd summarize -# 2. Copy the example config and fill in your keys -cp appsettings.example.json appsettings.json -nano appsettings.json # or your editor of choice +# Edit appsettings.json with your LLM endpoint/key (see Configuration below) +nano appsettings.json -# 3. Restore packages dotnet restore - -# 4. Run dotnet run ``` --- -## Google Cloud Setup (YouTube API Key) +## Using it -1. Go to [console.cloud.google.com](https://console.cloud.google.com) -2. Create or select a project -3. **APIs & Services → Library** → search "YouTube Data API v3" → Enable -4. **APIs & Services → Credentials → Create Credentials → API key** -5. (Optional but recommended) Restrict the key to only the YouTube Data API v3 +The app loops on a single prompt: -> Free quota: **10,000 units/day**. Each video lookup costs ~3 units. You can summarize thousands of videos before hitting the limit. +``` +Enter a YouTube URL or a local file path (or 'q' to quit): +``` + +- **YouTube**: paste any `watch?v=...`, `youtu.be/...`, `/shorts/...`, or + `/embed/...` URL (or a bare 11-character video ID). +- **Local document**: paste or type a path to an existing file. Supported + types: `.txt` `.md` `.csv` `.pdf` `.docx` `.odt` `.xlsx`. The kind of input + is auto-detected — no need to say which one it is. +- `q` quits. + +For each input you're asked: + +1. **Save transcript to file?** — `y/n`, defaults to **yes** on Enter. + Saves the extracted transcript/document text plus the summary to + `~/Downloads/transcripts/`. +2. **Choose summary mode** — an arrow-key list, Standard pre-selected: + - **Standard** — detailed bullet-point summary (the default; Enter picks it) + - **Personal Filter** — a 1–2 sentence summary plus an ACT / MONITOR / + IGNORE relevance verdict against personal priorities (time, finances, + health, family, service to others) + - **Custom** — write your own instructions on the spot (multi-line; finish + with a blank line). Replaces the built-in prompt entirely, for when you + want something other than a summary or a verdict — a table, an + extraction task, a specific format, etc. + +Ctrl+C cancels the current operation without killing the app. The terminal +tab title tracks progress (`Summarize` while idle, `Summarize ` while +working on something). --- ## Configuration Reference +All settings live in `appsettings.json` (bind to `AppSettings` in +[AppSettings.cs](AppSettings.cs)): + | Key | Description | Default | -|---|---|---| -| `YouTube:ApiKey` | Your YouTube Data API v3 key | *(required)* | -| `LLM:BaseUrl` | API endpoint | `http://localhost:11434/v1` | -| `LLM:ApiKey` | API key (any for Ollama) | `ollama` | -| `LLM:Model` | Chat model to use | `qwen3:14b` | -| `LLM:MaxTokens` | Max tokens in summary response | `1500` | -| `LLM:TimeoutSeconds` | Max time for LLM generation | `300` | -| `Summarizer:ChunkWordLimit` | Words per chunk for long videos | `3000` | -| `Summarizer:ShowTranscript` | Print raw transcript before summary | `false` | +| --- | --- | --- | +| `LLM:BaseUrl` | API endpoint — `https://api.openai.com/v1` for OpenAI, `http://localhost:11434/v1` for Ollama | `https://api.openai.com/v1` | +| `LLM:ApiKey` | API key (any non-empty value works for Ollama) | *(empty — required for OpenAI)* | +| `LLM:Model` | Chat model, e.g. `gpt-4o-mini` (OpenAI) or `qwen3:14b` (Ollama) | `gpt-4o-mini` | +| `LLM:MaxTokens` | Max tokens in the summary response | `1500` | +| `LLM:TimeoutSeconds` | Max time to wait per API call | `100` | +| `Summarizer:ChunkWordLimit` | Word count above which a transcript/document is split into chunks (map-reduce, see below). Must stay comfortably above ~200 — the fixed chunk overlap — or chunking will error on short inputs. | `3000` | +| `Summarizer:ShowTranscript` | Print the full extracted text before summarizing | `false` | + +Any value can be overridden with an environment variable using `__` as the +section separator (handy for CI/containers): + +```bash +export LLM__ApiKey="sk-..." +export LLM__Model="gpt-4o" +dotnet run +``` --- @@ -62,52 +102,55 @@ dotnet run ``` Program.cs -│ Main loop → parses URL → calls pipeline +│ Main loop → auto-detects YouTube URL vs. local file path │ -├── YouTubeService +├── YouTubeService — shells out to yt-dlp │ ├── ExtractVideoId() — URL parsing -│ ├── GetVideoMetadataAsync() — YouTube Data API v3 (Videos.list) -│ └── GetTranscriptAsync() — Caption list + timedtext download +│ ├── GetVideoMetadataAsync() — video title/channel/date/duration +│ └── GetTranscriptAsync() — caption download + VTT/SRT/timedtext parsing +│ +├── DocumentService — reads a local file +│ ├── BuildMetadata() — title/type/modified-date from the file +│ └── ExtractTextAsync() — dispatches by extension: +│ .txt/.md/.csv → read directly +│ .pdf → PdfPig +│ .docx/.xlsx → DocumentFormat.OpenXml +│ .odt → hand-rolled zip + XML (ODF content.xml) +│ +│ Both feed the same VideoMetadata / VideoTranscript shapes into: │ ├── SummarizerService -│ ├── SummarizeAsync() — Routes to single-pass or chunked -│ ├── SinglePassSummarize() — One OpenAI call for short videos -│ └── ChunkedSummarize() — Map-reduce for long videos +│ ├── SummarizeAsync() — routes to single-pass or chunked, per mode +│ │ (Standard / Personal Filter / Custom) +│ ├── SinglePassSummarize() — one LLM call for short inputs +│ └── ChunkedSummarizeAsync() — map-reduce for long inputs │ -└── ConsoleRenderer — All terminal output / formatting +├── TranscriptFileService — saves transcript + summary to a .txt file +│ +└── ConsoleRenderer — all terminal output (Spectre.Console): + banner, prompts, progress spinners, and + the final summary panel/grid ``` -### Caption Quality Transparency +### Source Quality Transparency -The app tracks how the transcript was obtained and flags it accordingly: +The app tracks how the text was obtained and flags it accordingly: -| Source | Label | Warning shown? | -|---|---|---| +| Source | Badge | Warning shown? | +| --- | --- | --- | | Owner-published captions | `✓ Owner-published` | No | -| Community-contributed | `✓ Community captions` | Minor note | -| Auto-generated (ASR) | `~ Auto-generated` | Yes — accuracy caveat | +| Community-contributed captions | `✓ Community captions` | Minor note | +| Auto-generated captions (ASR) | `~ Auto-generated` | Yes — accuracy caveat | | No captions (metadata only) | `✗ Metadata only` | Yes — limited accuracy | +| Local document | `✓ Local document` | No | -### Long Video Strategy +### Long-Input Strategy -Videos with transcripts exceeding `ChunkWordLimit` words use a **map-reduce** approach: - -1. **Split** — transcript divided into overlapping chunks (200-word overlap preserves context at boundaries) -2. **Map** — each chunk summarized independently -3. **Reduce** — chunk summaries combined into a final coherent summary - -This handles hour-long lectures, conference talks, and podcasts without hitting model context limits. - ---- - -## Environment Variable Overrides - -You can override `appsettings.json` values with environment variables, useful for CI or Docker: - -```bash -export YouTube__ApiKey="your-key" -export LLM__ApiKey="ollama" -dotnet run -``` - -Note the double-underscore `__` as the section separator (standard .NET configuration convention). +Transcripts/documents exceeding `ChunkWordLimit` words use a **map-reduce** +approach — split into overlapping chunks (200-word overlap preserves context +at boundaries), each chunk summarized independently, then combined into one +coherent result. This applies to every mode, including Custom (the combine +step gets an auto-generated wrapper prompt asking the model to merge the +partial responses while still honoring your instructions). Handles +hour-long videos, long reports, and multi-sheet spreadsheets without hitting +model context limits. diff --git a/SummarizerService.cs b/SummarizerService.cs index 3a92254..6dbf38b 100644 --- a/SummarizerService.cs +++ b/SummarizerService.cs @@ -117,19 +117,44 @@ public sealed class SummarizerService /// Produces a <see cref="VideoSummary"/> from the video's metadata and transcript. /// Automatically routes to single-pass or chunked strategy based on word count. /// </summary> + /// <param name="customPrompt"> + /// Required when <paramref name="mode"/> is <see cref="SummaryMode.Custom"/> — the + /// user's own instructions, used verbatim as the system prompt in place of the + /// built-in Standard/Personal Filter prompts. Ignored otherwise. + /// </param> public async Task<VideoSummary> SummarizeAsync( VideoMetadata metadata, VideoTranscript transcript, SummaryMode mode = SummaryMode.Standard, + string? customPrompt = null, CancellationToken ct = default) { string summaryText; // Select prompt set based on mode - var chunkPrompt = mode == SummaryMode.PersonalFilter - ? PersonalFilterSystemPrompt : ChunkSystemPrompt; - var combinePrompt = mode == SummaryMode.PersonalFilter - ? PersonalFilterCombinePrompt : CombineSystemPrompt; + string chunkPrompt, combinePrompt; + if (mode == SummaryMode.Custom) + { + if (string.IsNullOrWhiteSpace(customPrompt)) + throw new ArgumentException("A custom prompt is required when mode is Custom.", nameof(customPrompt)); + + chunkPrompt = customPrompt; + combinePrompt = + "You are combining several partial responses that were each produced " + + $"following these instructions:\n\n{customPrompt}\n\n" + + "Merge them into a single coherent response that still satisfies these " + + "instructions, without duplicating information that appears in multiple parts."; + } + else if (mode == SummaryMode.PersonalFilter) + { + chunkPrompt = PersonalFilterSystemPrompt; + combinePrompt = PersonalFilterCombinePrompt; + } + else + { + chunkPrompt = ChunkSystemPrompt; + combinePrompt = CombineSystemPrompt; + } if (transcript.WordCount <= _summarizerSettings.ChunkWordLimit) { diff --git a/TranscriptFileService.cs b/TranscriptFileService.cs index f9e5ca0..94ec2bd 100644 --- a/TranscriptFileService.cs +++ b/TranscriptFileService.cs @@ -29,19 +29,28 @@ public static class TranscriptFileService var fileName = $"{safeTitle}_{metadata.VideoId}.txt"; var filePath = Path.Combine(outputDirectory, fileName); + var isDocument = transcript.Source == TranscriptSource.LocalDocument; var sb = new StringBuilder(); // ── Metadata section ───────────────────────────────────────────────── sb.AppendLine("════════════════════════════════════════════════════════════════"); - sb.AppendLine(" VIDEO METADATA"); + sb.AppendLine(isDocument ? " DOCUMENT METADATA" : " VIDEO METADATA"); sb.AppendLine("════════════════════════════════════════════════════════════════"); sb.AppendLine(); sb.AppendLine($" Title: {metadata.Title}"); - sb.AppendLine($" Channel: {metadata.ChannelTitle}"); - sb.AppendLine($" Published: {metadata.PublishedAt:MMMM d, yyyy}"); - sb.AppendLine($" Duration: {metadata.FormattedDuration}"); - sb.AppendLine($" Video ID: {metadata.VideoId}"); - sb.AppendLine($" URL: https://youtu.be/{metadata.VideoId}"); + sb.AppendLine(isDocument ? $" Type: {metadata.ChannelTitle}" : $" Channel: {metadata.ChannelTitle}"); + sb.AppendLine(isDocument ? $" Modified: {metadata.PublishedAt:MMMM d, yyyy}" : $" Published: {metadata.PublishedAt:MMMM d, yyyy}"); + + if (metadata.Duration is not null) + sb.AppendLine($" Duration: {metadata.FormattedDuration}"); + + if (isDocument) + sb.AppendLine($" Location: {metadata.SourcePath ?? metadata.Title}"); + else + { + sb.AppendLine($" Video ID: {metadata.VideoId}"); + sb.AppendLine($" URL: https://youtu.be/{metadata.VideoId}"); + } if (!string.IsNullOrWhiteSpace(summaryText)) { @@ -63,6 +72,7 @@ public static class TranscriptFileService TranscriptSource.CommunityContributed => "Community-contributed captions", TranscriptSource.AutoGenerated => "Auto-generated (ASR)", TranscriptSource.MetadataOnly => "Metadata only (no captions)", + TranscriptSource.LocalDocument => "Local document", _ => "Unknown" }; sb.AppendLine($" Transcript Source: {sourceLabel}"); diff --git a/VideoModels.cs b/VideoModels.cs index e510328..fae8494 100644 --- a/VideoModels.cs +++ b/VideoModels.cs @@ -28,6 +28,12 @@ public sealed class VideoMetadata /// <summary>First 5000 characters of the video description (API cap).</summary> public string? Description { get; init; } + /// <summary> + /// Full path to the source file when this metadata describes a local + /// document rather than a YouTube video. Null for videos. + /// </summary> + public string? SourcePath { get; init; } + /// <summary>Human-readable duration parsed from <see cref="Duration"/>.</summary> public string FormattedDuration => Duration is null ? "Unknown" @@ -120,7 +126,10 @@ public enum TranscriptSource AutoGenerated, /// <summary>No captions available; summary based on metadata/description only.</summary> - MetadataOnly + MetadataOnly, + + /// <summary>Text extracted from a local document (PDF, Word, plain text, etc).</summary> + LocalDocument } /// <summary> @@ -136,7 +145,13 @@ public enum SummaryMode /// evaluation against personal priorities (time, finances, health, family, /// service to others), and a single-word verdict: ACT, MONITOR, or IGNORE. /// </summary> - PersonalFilter + PersonalFilter, + + /// <summary> + /// User-supplied instructions replace the built-in system prompt entirely, + /// for when a plain summary or ACT/MONITOR/IGNORE verdict isn't the goal. + /// </summary> + Custom } /// <summary> diff --git a/YoutubeSummarizer.csproj b/YoutubeSummarizer.csproj index 01e9738..61b0b99 100644 --- a/YoutubeSummarizer.csproj +++ b/YoutubeSummarizer.csproj @@ -17,6 +17,7 @@ <ItemGroup> <!-- OpenAI .NET SDK (official) --> + <PackageReference Include="DocumentFormat.OpenXml" Version="3.5.1" /> <PackageReference Include="OpenAI" Version="2.1.0" /> <!-- Microsoft.Extensions for config/DI patterns without full host overhead --> @@ -28,6 +29,7 @@ <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.0" /> <!-- Http provides AddHttpClient() / IHttpClientFactory --> <PackageReference Include="Microsoft.Extensions.Http" Version="10.0.0" /> + <PackageReference Include="PdfPig" Version="0.1.16" /> <!-- Polly for resilient HTTP retry logic --> <PackageReference Include="Polly" Version="8.4.1" /> -- 2.43.7