#!/usr/bin/env jruby # frozen_string_literal: true # # fetch_podcasts.rb - Podcast subscription management and episode fetching # for the liquidsoap radio automation stack (JRuby/Sequel variant). # # Concurrency: exclusive non-blocking File.lock (state/radio.lock) shared with # update_playlists.rb; skips cleanly if the sibling holds it. Databases run in # WAL mode with a busy timeout as a second safety net. # # Politeness: audio/video verdict cached in shows.media_class; gpodder OPML # pull uses bounded retry with exponential backoff + jitter. # --------------------------------------------------------------------------- # Gem path bootstrap: pin GEM_HOME/GEM_PATH before any require so the script # finds its gems regardless of how it was launched (sudo strips them by # default via env_reset). Derived from this file's own location so the # scripts are relocatable. We PREPEND our .gems dir to the existing GEM_PATH # rather than replacing it, so JRuby's shared/stdlib path stays reachable. # Guards prevent overriding an explicit environment. # --------------------------------------------------------------------------- SCRIPT_DIR = File.expand_path(File.dirname(__FILE__)) GEMS_DIR = File.join(SCRIPT_DIR, ".gems") ENV["GEM_HOME"] = GEMS_DIR unless ENV["GEM_HOME"] _existing_gem_path = ENV["GEM_PATH"].to_s.split(":").reject(&:empty?) ENV["GEM_PATH"] = ([GEMS_DIR] + _existing_gem_path).uniq.join(":") Gem.paths = { "GEM_HOME" => ENV["GEM_HOME"], "GEM_PATH" => ENV["GEM_PATH"] } require "sequel" require "net/http" require "openssl" require "json" require "logger" require "time" require "fileutils" require "cgi" require "base64" require "securerandom" require "rexml/document" ROOT = SCRIPT_DIR CONFIG_PATH = File.join(ROOT, "config.json") AUDIO_EXTS = [".mp3", ".m4a"].freeze VIDEO_EXTS = [".mp4", ".mov", ".avi", ".webm", ".mkv"].freeze $state_dir = nil $subs_db_path = nil $played_db_path = nil $podcasts_dir = nil $logs_dir = nil $playlists_dir = nil $lock_file = nil def load_config JSON.parse(File.read(CONFIG_PATH)) end def init_paths! cfg = load_config storage = File.realpath(cfg["storage"]) $state_dir = File.join(storage, "state") $subs_db_path = File.join($state_dir, "subscriptions.db") $played_db_path = File.join($state_dir, "played.db") $podcasts_dir = File.join(storage, "podcasts") $logs_dir = File.join(storage, "logs") $playlists_dir = File.join(storage, "playlists") $lock_file = File.join($state_dir, "radio.lock") end $log = Logger.new(STDOUT) $log.formatter = proc { |msg, _severity, _time, _progname| "#{Time.now} [INFO] #{msg}\n" } def log_error(msg) $log.error(msg) end def setup_logging! $log.instance_variable_set(:@logdev, Logger::LogDevice.new([STDOUT, File.join($logs_dir, "fetch.log")])) end # --------------------------------------------------------------------------- # Mutual exclusion via flock on a shared lockfile. # --------------------------------------------------------------------------- $lock_fh = nil def acquire_lock! Dir.mkdir($state_dir) unless Dir.exist?($state_dir) fh = File.open($lock_file, File::RDWR | File::CREAT, 0o644) begin fh.flock(File::LOCK_EX | File::LOCK_NB) rescue Errno::EACCES, Errno::EAGAIN fh.close return false end fh.truncate(0) fh.write(Process.pid.to_s) fh.rewind $lock_fh = fh true end def release_lock! return if $lock_fh.nil? begin $lock_fh.flock(File::LOCK_UN) $lock_fh.close ensure $lock_fh = nil end end # --------------------------------------------------------------------------- # Schema: single source of truth, applied idempotently via raw SQL through # Database#execute. Raw DDL avoids the Sequel create_table/alter_table DSL, # which is unreliable under JRuby (instance_exec'd generator methods can # resolve to nil). Works identically on CRuby and JDBC. # --------------------------------------------------------------------------- SHOWS_SQL = <<~SQL.freeze CREATE TABLE IF NOT EXISTS shows ( slug TEXT PRIMARY KEY, guid TEXT NOT NULL UNIQUE, name TEXT NOT NULL, feed_url TEXT NOT NULL UNIQUE, source TEXT DEFAULT 'manual', opml_import INTEGER DEFAULT 0, archived INTEGER DEFAULT 1, media_class TEXT, created_at TEXT ); SQL EPISODES_SQL = <<~SQL.freeze CREATE TABLE IF NOT EXISTS episodes ( id INTEGER PRIMARY KEY AUTOINCREMENT, show_slug TEXT NOT NULL, guid TEXT NOT NULL, title TEXT, file_path TEXT, enclosure_url TEXT, runlength INTEGER, played INTEGER DEFAULT 0, played_at TEXT, UNIQUE (show_slug, guid) ); SQL INDEX_EPISODES_SQL = <<~SQL.freeze CREATE INDEX IF NOT EXISTS idx_episodes_show_played ON episodes (show_slug, played); SQL def tune(db) # Plain-SQL pragmas via Database#execute, which works on CRuby and JRuby/JDBC # across all modern Sequel versions (the :pragma extension is CRuby-only and # db.sql requires Sequel >= 5.42). db.execute("PRAGMA journal_mode=WAL;") db.execute("PRAGMA busy_timeout=5000;") end def connect_subs db = Sequel.connect("jdbc:sqlite:#{$subs_db_path}") tune(db) db.execute(SHOWS_SQL) db end def connect_played db = Sequel.connect("jdbc:sqlite:#{$played_db_path}") tune(db) db.execute(EPISODES_SQL) db.execute(INDEX_EPISODES_SQL) db end # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def slugify(name) s = name.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/\A_+|_+\z/, "") s = "show" if s.empty? s[0, 60] end def gen_uuid SecureRandom.uuid end def classify_feed(entries) return "unknown" if entries.nil? || entries.empty? entry = entries.first enclosures = entry[:enclosures] || [] return "unknown" if enclosures.empty? mime = (enclosures.first[:type] || "").downcase return "audio" if mime.start_with?("audio/") return "video" if mime.start_with?("video/") url = (enclosures.first[:href] || "").downcase return "audio" if AUDIO_EXTS.any? { |ext| url.end_with?(ext) } return "video" if VIDEO_EXTS.any? { |ext| url.end_with?(ext) } "unknown" end def get_media_class(db, slug) row = db[:shows].where(slug: slug).first row ? row[:media_class] : nil end def set_media_class(db, slug, cls) db[:shows].where(slug: slug).update(media_class: cls) end # --------------------------------------------------------------------------- # Feed parsing (Net::HTTP + REXML) # --------------------------------------------------------------------------- def fetch_feed(feed_url) uri = URI.parse(feed_url) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = (uri.scheme == "https") http.timeout = 60 resp = http.request(Net::HTTP::Get.new(uri.request_uri)) return nil unless resp.is_a?(Net::HTTPSuccess) doc = REXML::Document.new(resp.body) channel = doc.elements["//channel"] return nil if channel.nil? title = channel.elements["title"]&.text entries = [] channel.get_elements("./item").each do |item| link_el = item.elements["link"] title_el = item.elements["title"] guid_el = item.elements["guid"] dur_el = item.elements["media:duration"] || item.elements["itunes:duration"] enc_el = item.elements["enclosure"] enclosures = [] if enc_el enclosures << { href: enc_el.attributes["url"], type: enc_el.attributes["type"], length: enc_el.attributes["length"] } end entries << { link: link_el&.text, title: title_el&.text, guid: guid_el&.text, enclosures: enclosures, duration: dur_el&.text } end { title: title, entries: entries } rescue StandardError => e log_error("Feed parse error for #{feed_url}: #{e.message}") nil end # --------------------------------------------------------------------------- # gPodder sync with bounded retry + exponential backoff + jitter # --------------------------------------------------------------------------- def gpodder_sync(cfg) g = cfg["gpodder"] base = g["host"].chomp("/") username = g["username"] password = g["password"] path = "/subscriptions/#{CGI.escape(username)}.opml" puts "--- Syncing subscriptions from #{base} ---" puts "Fetching subscriptions for '#{username}'..." max_attempts = 4 backoff_base = 2.0 resp = nil max_attempts.times do |attempt| begin uri = URI.parse("#{base}#{path}") http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = (uri.scheme == "https") http.timeout = 60 req = Net::HTTP::Get.new(uri.request_uri) credentials = Base64.strict_encode64("#{username}:#{password}") req["Authorization"] = "Basic #{credentials}" req["User-Agent"] = "radio-automation/1.0" resp = http.request(req) break rescue StandardError => e if attempt == max_attempts - 1 log_error("gPodder sync failed after #{attempt + 1} attempts: #{e.class.name}") return [] end delay = (backoff_base ** (attempt + 1)) + rand $log.warn("gPodder request error (#{e.class.name}); retrying in #{delay.round(1)}s") sleep(delay) end end return [] if resp.nil? case resp.code.to_i when 200 body = resp.body return [] if body.strip.empty? parse_opml(body) when 401 log_error("gPodder sync failed: 401 Unauthorized. Check username/password in config.json.") when 404 log_error("gPodder sync failed: 404 Not Found. User may not exist or has no subscriptions.") when 429 log_error("gPodder sync throttled (429). Will retry next cycle.") else code = resp.code.to_i if code >= 500 log_error("gPodder sync server error (#{code}). Will retry next cycle.") else log_error("gPodder sync failed: unexpected response #{code}: #{resp.body[0, 200]}") end end [] end def parse_opml(xml_string) shows = [] doc = REXML::Document.new(xml_string) doc.elements.each("//outline") do |outline| feed_url = (outline.attributes["xmlUrl"] || "").strip name = (outline.attributes["text"] || "").strip guid = (outline.attributes["guid"] || "").strip next unless feed_url.match?(/\Ahttps?:\/\//) shows << { name: name, feed_url: feed_url, guid: guid.empty? ? nil : guid } end shows rescue REXML::ParseException => e log_error("Failed to parse OPML XML: #{e.message}") [] end # --------------------------------------------------------------------------- # Show registration / pruning # --------------------------------------------------------------------------- def register_remote_shows(remote_shows) db = connect_subs added = 0 skipped_video = 0 remote_shows.each do |show| slug = slugify(show[:name]) next if db[:shows].where(slug: slug).count > 0 parsed = fetch_feed(show[:feed_url]) if parsed.nil? $log.warn("Skipping '#{show[:name]}': could not fetch feed.") next end cls = classify_feed(parsed[:entries]) if cls == "video" $log.info("Skipping '#{show[:name]}' (#{slug}): video podcast, not audio.") skipped_video += 1 next end guid = show[:guid] || gen_uuid db[:shows].insert( slug: slug, guid: guid, name: show[:name], feed_url: show[:feed_url], source: "gpodder", opml_import: 0, archived: 1, media_class: cls ) $log.info("Registered new show: #{show[:name]} (#{slug}) [#{cls}]") added += 1 end db.disconnect $log.info("Filtered out #{skipped_video} video podcast(s).") if skipped_video > 0 added end def prune_stale_shows(remote_shows) db = connect_subs remote_slugs = remote_shows.map { |s| slugify(s[:name]) }.to_set stale = db[:shows].where(source: "gpodder", opml_import: 0).all removed = 0 stale.each do |row| next if remote_slugs.include?(row[:slug]) remove_show_data(row[:slug]) db[:shows].where(slug: row[:slug]).delete $log.info("Pruned stale show: #{row[:name]} (#{row[:slug]})") removed += 1 end db.disconnect removed end # --------------------------------------------------------------------------- # Episode extraction / download # --------------------------------------------------------------------------- def extract_duration(entry) dur = entry[:duration] if dur return dur.to_i if dur.match?(/\A\d+\z/) m = dur.match(/\APT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?\z/i) if m h = m[1] ? m[1].to_i : 0 mn = m[2] ? m[2].to_i : 0 s = m[3] ? m[3].to_i : 0 return h * 3600 + mn * 60 + s end end enc = entry[:enclosures]&.first if enc && enc[:length] bytes = enc[:length].to_i return (bytes * 8 / 128_000) if bytes > 0 end nil end def download_episode(url, dest_dir, filename) dest = File.join(dest_dir, filename) return dest if File.exist?(dest) uri = URI.parse(url) http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = (uri.scheme == "https") http.timeout = 120 tmp = "#{dest}.part" begin http.request_get(uri.request_uri) do |response| raise "HTTP #{response.code}" unless response.is_a?(Net::HTTPSuccess) File.open(tmp, "wb") do |f| response.read_body { |chunk| f.write(chunk) } end end File.rename(tmp, dest) dest rescue StandardError => e log_error("Download failed for #{url}: #{e.message}") File.delete(tmp) if File.exist?(tmp) nil end end def safe_filename(title, fallback) name = (title || "").gsub(/[^\w\s.\-]/, "").strip.tr(" ", "_") name = fallback if name.empty? "#{name[0, 120]}.mp3" end def show_archived?(db, slug) row = db[:shows].where(slug: slug).first row.nil? ? true : row[:archived] == 1 end # --------------------------------------------------------------------------- # Per-show episode fetch # --------------------------------------------------------------------------- def fetch_show_episodes(slug, name, feed_url) dest_dir = File.join($podcasts_dir, slug) Dir.mkdir(dest_dir) unless Dir.exist?(dest_dir) subs_db = connect_subs cached_cls = get_media_class(subs_db, slug) if cached_cls == "video" subs_db.disconnect return 0 end parsed = fetch_feed(feed_url) if parsed.nil? subs_db.disconnect return 0 end if cached_cls.nil? cls = classify_feed(parsed[:entries]) set_media_class(subs_db, slug, cls) if cls == "video" $log.info("'#{name}' (#{slug}) classified as video; skipping.") subs_db.disconnect return 0 end end played_db = connect_played seen = played_db[:episodes].where(show_slug: slug).select_map(:guid).to_set archived = show_archived?(subs_db, slug) new_count = 0 parsed[:entries].each do |entry| guid = entry[:guid] || entry[:link] || entry[:title] || "" next if seen.include?(guid) enclosures = entry[:enclosures] || [] next if enclosures.empty? mime = (enclosures.first[:type] || "").downcase next if mime.start_with?("video/") audio_url = enclosures.first[:href] next if audio_url.nil? || audio_url.empty? title = entry[:title] || "untitled" duration = extract_duration(entry) if archived filename = safe_filename(title, guid[-20..]) file_path = download_episode(audio_url, dest_dir, filename) next if file_path.nil? insert_episode(played_db, slug, guid, title, file_path, audio_url, duration) else insert_episode(played_db, slug, guid, title, nil, audio_url, duration) end new_count += 1 kind = archived ? "downloaded" : "live" $log.info(" New episode: #{title} [#{kind}]") end played_db.disconnect subs_db.disconnect new_count end def insert_episode(db, slug, guid, title, file_path, enclosure_url, duration) # INSERT OR IGNORE semantics via the UNIQUE(show_slug, guid) constraint. db.transaction do db[:episodes].insert( show_slug: slug, guid: guid, title: title, file_path: file_path, enclosure_url: enclosure_url, runlength: duration, played: 0 ) end rescue Sequel::UniqueConstraintViolation # Already recorded; ignore. end def fetch_all_episodes db = connect_subs shows = db[:shows].order(:name).all db.disconnect total_new = 0 shows.each do |show| $log.info("--- Fetching: #{show[:name]} (#{show[:slug]}) ---") begin n = fetch_show_episodes(show[:slug], show[:name], show[:feed_url]) total_new += n rescue StandardError => e log_error("Unexpected error fetching #{show[:slug]}: #{e.message}") end end $log.info("=== Fetch complete: #{total_new} new episode(s) ===") end # --------------------------------------------------------------------------- # Administrative commands # --------------------------------------------------------------------------- def list_shows(detail: false) db = connect_subs rows = db[:shows].order(:name).all db.disconnect if rows.empty? puts "No shows registered." return end puts format("%-30s %-10s %-8s %-10s %s", "SLUG", "ARCHIVED", "MEDIA", "SOURCE", "NAME") rows.each do |r| arch = r[:archived] == 1 ? "yes" : "no" media = r[:media_class] || "?" line = format("%-30s %-10s %-8s %-10s %s", r[:slug], arch, media, r[:source], r[:name]) line += "\n" + (" " * 50) + r[:feed_url] if detail puts line end end def add_show(feed_url) parsed = fetch_feed(feed_url) if parsed.nil? || parsed[:title].nil? log_error("Could not determine show title from #{feed_url}") return end cls = classify_feed(parsed[:entries]) if cls == "video" log_error("Refusing to add '#{parsed[:title]}': video podcast detected.") return end name = parsed[:title] slug = slugify(name) guid = gen_uuid db = connect_subs begin db[:shows].insert( slug: slug, guid: guid, name: name, feed_url: feed_url, source: "manual", opml_import: 0, archived: 1, media_class: cls ) rescue Sequel::UniqueConstraintViolation # already present end db.disconnect $log.info("Added show: #{name} (#{slug}) [#{cls}]") fetch_show_episodes(slug, name, feed_url) end def set_archive(slug, value) db = connect_subs row = db[:shows].where(slug: slug).first if row.nil? log_error("No show found with slug '#{slug}'.") db.disconnect return end db[:shows].where(slug: slug).update(archived: value) db.disconnect state = value == 1 ? "archived" : "non-archived (live)" $log.info("Show '#{row[:name]}' (#{slug}) is now #{state}.") end def remove_show_data(slug) pod_dir = File.join($podcasts_dir, slug) FileUtils.rm_rf(pod_dir) if Dir.exist?(pod_dir) txt = File.join($playlists_dir, "#{slug}.txt") File.delete(txt) if File.exist?(txt) end def delete_show(slug) db = connect_subs row = db[:shows].where(slug: slug).first if row.nil? log_error("No show found with slug '#{slug}'.") db.disconnect return end remove_show_data(slug) db[:shows].where(slug: slug).delete db.disconnect played_db = connect_played played_db[:episodes].where(show_slug: slug).delete played_db.disconnect $log.info("Deleted show: #{row[:name]} (#{slug})") end def import_opml(path) content = File.read(path) shows = parse_opml(content) db = connect_subs added = 0 skipped_video = 0 shows.each do |show| slug = slugify(show[:name]) next if db[:shows].where(slug: slug).count > 0 parsed = fetch_feed(show[:feed_url]) if parsed.nil? $log.warn("OPML import: skipping '#{show[:name]}', could not fetch feed.") next end cls = classify_feed(parsed[:entries]) if cls == "video" $log.info("OPML import: skipping '#{show[:name]}' (#{slug}): video podcast.") skipped_video += 1 next end guid = show[:guid] || gen_uuid db[:shows].insert( slug: slug, guid: guid, name: show[:name], feed_url: show[:feed_url], source: "opml", opml_import: 1, archived: 1, media_class: cls ) added += 1 end db.disconnect $log.info("OPML import: #{added} added, #{skipped_video} video shows filtered out.") end def run_fetch(config) g = config["gpodder"] if g["enable"] == true remote = gpodder_sync(config) if remote.empty? $log.warn("No subscriptions retrieved from gPodder; using local registry only.") else added = register_remote_shows(remote) pruned = prune_stale_shows(remote) $log.info("Sync: #{added} added, #{pruned} pruned.") end end fetch_all_episodes end def main args = ARGV.dup option = args.shift init_paths! [$state_dir, $logs_dir, $podcasts_dir, $playlists_dir].each do |dir| Dir.mkdir(dir) unless Dir.exist?(dir) end setup_logging! config = load_config case option when "--list-shows" list_shows(detail: args.include?("--detail")) when "--add-show" add_show(args.first) when "--delete-show" delete_show(args.first) when "--archive" set_archive(args.first, 1) when "--unarchive" set_archive(args.first, 0) when "--import-opml" import_opml(args.first) else if !acquire_lock! $log.info("Another radio process holds the lock; skipping this run.") return end begin run_fetch(config) ensure release_lock! end end end main if __FILE__ == $PROGRAM_NAME