#!/usr/bin/env ruby # frozen_string_literal: true # sync_docs_discourse.rb # # Syncs sunnypilot documentation from docs_sp/ to a Discourse forum. # Reads raw .md files, converts MkDocs Material syntax to Discourse-compatible # markdown (Obsidian-style callouts), resolves internal links to Discourse URLs, # and creates or updates topics via the Discourse API. # # Usage: # ruby sync_docs_discourse.rb [--dry-run] [--verbose] # # Environment variables (required unless --dry-run): # DISCOURSE_URL - Base URL of the Discourse instance (e.g. https://forum.sunnypilot.ai) # DISCOURSE_API_KEY - API key with topic create/update permissions # DISCOURSE_API_USER - Username for API requests (e.g. system or a bot account) # DISCOURSE_CATEGORY - Category slug or ID for documentation topics (default: "documentation") # # Optional: # DOCS_BASE_URL - Base URL for the static docs site (default: https://docs.sunnypilot.ai) require "net/http" require "uri" require "json" require "yaml" require "digest" require "fileutils" require "optparse" # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- DOCS_DIR = File.expand_path("../../", __FILE__) # docs_sp/ MKDOCS_YML = File.expand_path("../../../mkdocs-sp.yml", __FILE__) CACHE_DIR = File.expand_path("../../../.discourse_sync_cache", __FILE__) DOCS_BASE_URL = ENV.fetch("DOCS_BASE_URL", "https://docs.sunnypilot.ai") DISCOURSE_URL = ENV["DISCOURSE_URL"] DISCOURSE_API_KEY = ENV["DISCOURSE_API_KEY"] DISCOURSE_API_USER = ENV.fetch("DISCOURSE_API_USER", "system") DISCOURSE_CATEGORY = ENV.fetch("DISCOURSE_CATEGORY", "documentation") # MkDocs admonition type → Obsidian/Discourse callout type ADMONITION_MAP = { "note" => "NOTE", "abstract" => "ABSTRACT", "info" => "INFO", "tip" => "TIP", "success" => "SUCCESS", "question" => "QUESTION", "warning" => "WARNING", "failure" => "FAILURE", "danger" => "DANGER", "bug" => "BUG", "example" => "EXAMPLE", "quote" => "QUOTE", }.freeze # Pages to skip (not meaningful as standalone Discourse topics) SKIP_FILES = %w[ index.md README.md ].freeze # --------------------------------------------------------------------------- # CLI Options # --------------------------------------------------------------------------- options = { dry_run: false, verbose: false } OptionParser.new do |opts| opts.banner = "Usage: #{$PROGRAM_NAME} [options]" opts.on("--dry-run", "Show what would be synced without making API calls") { options[:dry_run] = true } opts.on("--verbose", "Print detailed conversion output") { options[:verbose] = true } opts.on("-h", "--help", "Show this help") { puts opts; exit } end.parse! DRY_RUN = options[:dry_run] VERBOSE = options[:verbose] unless DRY_RUN %w[DISCOURSE_URL DISCOURSE_API_KEY].each do |var| abort "Error: #{var} environment variable is required" unless ENV[var] end end # --------------------------------------------------------------------------- # MkDocs → Discourse Markdown Converter # --------------------------------------------------------------------------- module MkDocsConverter module_function # Main entry point: convert a full MkDocs Material markdown string # to Discourse-compatible markdown. def convert(content, file_path:, nav_slug_map: {}) result = content.dup # 1. Strip YAML front matter result = strip_front_matter(result) # 2. Convert admonitions (!!! type "title" / ??? type "title") result = convert_admonitions(result) # 3. Convert Material tabs (=== "Tab Name") result = convert_tabs(result) # 4. Convert grid cards result = convert_grid_cards(result) # 5. Convert Material emoji shortcodes to Unicode or strip result = convert_emoji_shortcodes(result) # 6. Resolve internal .md links to docs site URLs result = resolve_internal_links(result, file_path: file_path) # 7. Clean up excessive blank lines result = result.gsub(/\n{4,}/, "\n\n\n") result.strip + "\n" end # Remove YAML front matter (--- ... ---) def strip_front_matter(content) content.sub(/\A---\n.*?\n---\n*/m, "") end # Convert MkDocs admonitions to Obsidian/Discourse callouts. # # Input: # !!! warning "Title" # Content line 1 # Content line 2 # # Output: # > [!WARNING] Title # > Content line 1 # > Content line 2 # # Also handles collapsible (??? / ???+) variants. def convert_admonitions(content) lines = content.lines result = [] i = 0 while i < lines.length line = lines[i] # Match admonition opener: !!! type "title" or ??? type "title" or ???+ type "title" if line =~ /^(\s*)(\!{3}|\?{3}\+?) (\w+)(?: "([^"]*)")?/ indent = $1 marker = $2 ad_type = $3.downcase title = $4 callout_type = ADMONITION_MAP[ad_type] || ad_type.upcase # Build the callout header header = "#{indent}> [!#{callout_type}]" header += " #{title}" if title && !title.empty? # For collapsible (???), add a note if marker.start_with?("???") collapsed = !marker.include?("+") header += " *(click to #{collapsed ? 'expand' : 'collapse'})*" if title.nil? || title.empty? end result << header + "\n" i += 1 # Collect indented content lines (4 spaces deeper than the opener) content_indent = indent + " " while i < lines.length content_line = lines[i] if content_line =~ /^#{Regexp.escape(content_indent)}/ # Indented content line — part of the admonition stripped = content_line.sub(/^#{Regexp.escape(content_indent)}/, "") result << "#{indent}> #{stripped}" i += 1 elsif content_line.strip.empty? # Blank line: only part of admonition if the next non-blank # line is still indented at content level j = i + 1 j += 1 while j < lines.length && lines[j].strip.empty? if j < lines.length && lines[j] =~ /^#{Regexp.escape(content_indent)}/ result << "#{indent}>\n" i += 1 else # Blank line ends the admonition break end else break end end else result << line i += 1 end end result.join end # Convert Material tabs to Discourse-friendly headings with horizontal rules. # # Input: # === "Tab Name" # Content # # Output: # **Tab Name** # # Content # # --- def convert_tabs(content) lines = content.lines result = [] i = 0 while i < lines.length line = lines[i] if line =~ /^(\s*)=== "([^"]+)"/ indent = $1 tab_name = $2 result << "#{indent}**#{tab_name}**\n" result << "\n" i += 1 # Collect indented content content_indent = indent + " " while i < lines.length content_line = lines[i] if content_line =~ /^#{Regexp.escape(content_indent)}/ || content_line.strip.empty? if content_line.strip.empty? result << "\n" else stripped = content_line.sub(/^#{Regexp.escape(content_indent)}/, "") result << "#{indent}#{stripped}" end i += 1 else break end end result << "#{indent}---\n" result << "\n" else result << line i += 1 end end result.join end # Convert grid cards to simple lists (Discourse doesn't support grid cards) def convert_grid_cards(content) content .gsub(/
/, "") .gsub(/<\/div>/, "") end # Convert Material emoji shortcodes to plain text or remove # e.g., :material-rocket-launch: → 🚀 (or just strip) EMOJI_MAP = { ":material-rocket-launch:" => "🚀", ":material-cog:" => "⚙️", ":material-car:" => "🚗", ":material-shield:" => "🛡️", }.freeze def convert_emoji_shortcodes(content) result = content.dup EMOJI_MAP.each do |shortcode, emoji| result.gsub!(shortcode, emoji) end # Strip any remaining :material-*: shortcodes result.gsub(/:material-[\w-]+:/, "") end # Resolve internal links: (../features/icbm.md) → (https://docs.sunnypilot.ai/features/icbm/) def resolve_internal_links(content, file_path:) content.gsub(/\]\(([^)]+\.md)\)/) do |match| relative_path = $1 # Skip external URLs next match if relative_path.start_with?("http") # Resolve the relative path from the current file's directory current_dir = File.dirname(file_path) resolved = File.expand_path(relative_path, current_dir) # Make it relative to docs_sp/ docs_relative = resolved.sub(%r{^.*/docs_sp/}, "") # Convert to URL path: remove .md, add trailing slash url_path = docs_relative.sub(/\.md$/, "/") "](#{DOCS_BASE_URL}/#{url_path})" end end end # --------------------------------------------------------------------------- # Discourse API Client # --------------------------------------------------------------------------- module DiscourseAPI module_function def base_uri URI.parse(DISCOURSE_URL) end def headers { "Content-Type" => "application/json", "Api-Key" => DISCOURSE_API_KEY, "Api-Username" => DISCOURSE_API_USER, } end # Look up category ID by slug def category_id(slug) uri = URI.join(DISCOURSE_URL, "/c/#{slug}/show.json") response = http_get(uri) return nil unless response.is_a?(Net::HTTPSuccess) data = JSON.parse(response.body) data.dig("category", "id") end # Search for an existing topic by external_id (stored in the topic's first post) # We use a convention: embed an HTML comment def find_topic_by_sync_id(sync_id) search_query = "" uri = URI.join(DISCOURSE_URL, "/search.json?q=#{URI.encode_www_form_component(search_query)}") response = http_get(uri) return nil unless response.is_a?(Net::HTTPSuccess) data = JSON.parse(response.body) topics = data.dig("topics") || [] topics.first end # Create a new topic def create_topic(title:, raw:, category_id:, tags: []) uri = URI.join(DISCOURSE_URL, "/posts.json") payload = { title: title, raw: raw, category: category_id, tags: tags, } http_post(uri, payload) end # Update an existing topic's first post def update_post(post_id:, raw:, edit_reason: "Documentation sync") uri = URI.join(DISCOURSE_URL, "/posts/#{post_id}.json") payload = { post: { raw: raw, edit_reason: edit_reason, }, } http_put(uri, payload) end # Get a topic's first post ID def first_post_id(topic_id) uri = URI.join(DISCOURSE_URL, "/t/#{topic_id}.json") response = http_get(uri) return nil unless response.is_a?(Net::HTTPSuccess) data = JSON.parse(response.body) data.dig("post_stream", "posts", 0, "id") end # --- HTTP helpers --- def http_get(uri) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == "https" request = Net::HTTP::Get.new(uri, headers) http.request(request) end def http_post(uri, payload) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == "https" request = Net::HTTP::Post.new(uri, headers) request.body = payload.to_json http.request(request) end def http_put(uri, payload) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == "https" request = Net::HTTP::Put.new(uri, headers) request.body = payload.to_json http.request(request) end end # --------------------------------------------------------------------------- # Nav Parser — extract title + path from mkdocs-sp.yml nav # --------------------------------------------------------------------------- module NavParser module_function # Parse the mkdocs nav structure into a flat list of { title:, path: } def parse(mkdocs_yml_path) config = YAML.safe_load(File.read(mkdocs_yml_path)) nav = config["nav"] || [] flatten_nav(nav) end def flatten_nav(items, prefix_parts = []) result = [] items.each do |item| case item when Hash item.each do |key, value| case value when String # Skip external links next if value.start_with?("http") result << { title: key, path: value, breadcrumb: prefix_parts + [key] } when Array result.concat(flatten_nav(value, prefix_parts + [key])) end end when String # Bare path without title (unlikely in our nav) result << { title: File.basename(item, ".md").tr("-", " ").capitalize, path: item } end end result end end # --------------------------------------------------------------------------- # Content Cache — skip unchanged files # --------------------------------------------------------------------------- module ContentCache module_function def cache_path(file_path) slug = file_path.gsub("/", "_").gsub(".md", "") File.join(CACHE_DIR, "#{slug}.sha256") end def changed?(file_path, content_hash) cached = cache_path(file_path) return true unless File.exist?(cached) File.read(cached).strip != content_hash end def save(file_path, content_hash) FileUtils.mkdir_p(CACHE_DIR) File.write(cache_path(file_path), content_hash) end end # --------------------------------------------------------------------------- # Main Sync Logic # --------------------------------------------------------------------------- def sync_doc(entry, category_id) file_path = File.join(DOCS_DIR, entry[:path]) unless File.exist?(file_path) puts " ⚠ File not found: #{file_path}" return :skipped end raw_content = File.read(file_path, encoding: "utf-8") content_hash = Digest::SHA256.hexdigest(raw_content) # Skip unchanged files unless ContentCache.changed?(entry[:path], content_hash) puts " ✓ Unchanged: #{entry[:path]}" if VERBOSE return :unchanged end # Convert MkDocs → Discourse markdown converted = MkDocsConverter.convert(raw_content, file_path: file_path) # Prepend breadcrumb navigation if entry[:breadcrumb] && entry[:breadcrumb].length > 1 breadcrumb = entry[:breadcrumb][0..-2].join(" › ") converted = "*#{breadcrumb}*\n\n#{converted}" end # Append sync ID comment and docs site link docs_url = "#{DOCS_BASE_URL}/#{entry[:path].sub(/\.md$/, '/')}" converted += "\n\n---\n" converted += "*This documentation is automatically synced from the [sunnypilot docs site](#{docs_url}).*\n" converted += "\n" title = entry[:title] if DRY_RUN puts " → Would sync: \"#{title}\" (#{entry[:path]})" if VERBOSE puts " --- Converted content (first 500 chars) ---" puts " #{converted[0..500].gsub("\n", "\n ")}" puts " ---" end ContentCache.save(entry[:path], content_hash) # Cache even in dry-run for testing return :would_sync end # Check if topic already exists existing = DiscourseAPI.find_topic_by_sync_id(entry[:path]) if existing # Update existing topic post_id = DiscourseAPI.first_post_id(existing["id"]) if post_id response = DiscourseAPI.update_post(post_id: post_id, raw: converted) if response.is_a?(Net::HTTPSuccess) puts " ✓ Updated: \"#{title}\" (topic ##{existing['id']})" ContentCache.save(entry[:path], content_hash) return :updated else puts " ✗ Failed to update: #{response.code} #{response.body[0..200]}" return :error end else puts " ✗ Could not find first post for topic ##{existing['id']}" return :error end else # Create new topic response = DiscourseAPI.create_topic( title: "#{title} — sunnypilot Docs", raw: converted, category_id: category_id, tags: ["docs", "auto-sync"] ) if response.is_a?(Net::HTTPSuccess) || response.is_a?(Net::HTTPCreated) data = JSON.parse(response.body) puts " ✓ Created: \"#{title}\" (topic ##{data['topic_id']})" ContentCache.save(entry[:path], content_hash) return :created else puts " ✗ Failed to create: #{response.code} #{response.body[0..200]}" return :error end end end # --------------------------------------------------------------------------- # Entry Point # --------------------------------------------------------------------------- def main puts "sunnypilot Documentation → Discourse Sync" puts "=" * 50 puts "Mode: #{DRY_RUN ? 'DRY RUN' : 'LIVE'}" puts "Docs dir: #{DOCS_DIR}" puts "Discourse: #{DISCOURSE_URL || '(dry-run, no URL)'}" puts # Parse nav entries nav_entries = NavParser.parse(MKDOCS_YML) puts "Found #{nav_entries.length} nav entries" # Filter out skipped files nav_entries.reject! { |e| SKIP_FILES.include?(File.basename(e[:path])) } puts "After filtering: #{nav_entries.length} pages to sync" puts # Resolve category ID category_id = nil unless DRY_RUN category_id = DiscourseAPI.category_id(DISCOURSE_CATEGORY) abort "Error: Could not find category '#{DISCOURSE_CATEGORY}'" unless category_id puts "Category: #{DISCOURSE_CATEGORY} (ID: #{category_id})" puts end # Sync each page stats = { created: 0, updated: 0, unchanged: 0, skipped: 0, error: 0, would_sync: 0 } nav_entries.each_with_index do |entry, idx| # Rate limiting: 1 request per second for live mode sleep(1) if !DRY_RUN && idx > 0 result = sync_doc(entry, category_id) stats[result] = (stats[result] || 0) + 1 end # Summary puts puts "=" * 50 puts "Sync complete!" stats.each do |status, count| next if count == 0 puts " #{status}: #{count}" end end main