From 7a57591820985badf078e206dcef4803e83352d7 Mon Sep 17 00:00:00 2001 From: "G. Gibson" Date: Mon, 31 Aug 2026 11:21:36 -0700 Subject: [PATCH] audio-only opml filter --- fetch_podcasts.py | 78 +++++++++++++++++++++++++++++---- fetch_podcasts.rb | 108 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 164 insertions(+), 22 deletions(-) diff --git a/fetch_podcasts.py b/fetch_podcasts.py index c4627e5..18c6b01 100755 --- a/fetch_podcasts.py +++ b/fetch_podcasts.py @@ -5,6 +5,9 @@ for the liquidsoap radio automation stack. All data (podcasts, state DBs, logs, playlists) lives under the storage path defined in config.json ("storage" key), keeping the boot drive clean. + +Filters out video podcasts: only shows whose latest enclosure has an +audio/* MIME type are registered. """ import argparse @@ -105,6 +108,29 @@ def slugify(name): s = re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_") return s[:60] or "show" +def is_audio_feed(parsed): + """Check whether the feed's latest episode has an audio enclosure. + Returns True if audio, False if video or unknown. + """ + if not parsed.entries: + return True # No entries yet; let it through, will fail on fetch + entry = parsed.entries[0] + enclosures = entry.get("enclosures") or [] + if not enclosures: + return True # No enclosure info; assume audio + mime_type = (enclosures[0].get("type") or "").lower() + if mime_type.startswith("audio/"): + return True + if mime_type.startswith("video/"): + return False + # Unknown type: check URL extension as fallback + url = (enclosures[0].get("href") or "").lower() + if any(url.endswith(ext) for ext in AUDIO_EXTS): + return True + if any(url.endswith(ext) for ext in (".mp4", ".mov", ".avi", ".webm", ".mkv")): + return False + return True # Default to allowing if undetermined + def gpodder_sync(cfg): g = cfg["gpodder"] base = g["host"].rstrip("/") @@ -156,18 +182,31 @@ def parse_opml(xml_string): def register_remote_shows(remote_shows): db = open_subs_db() added = 0 + skipped_video = 0 for show in remote_shows: slug = slugify(show["name"]) existing = db.execute("SELECT slug FROM shows WHERE slug = ?", (slug,)).fetchone() - if existing is None: - db.execute( - "INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)", - (slug, show["name"], show["feed_url"]), - ) - log.info("Registered new show: %s (%s)", show["name"], slug) - added += 1 + if existing is not None: + continue + # Audio-only filter: check the feed before registering + parsed = fetch_feed(show["feed_url"]) + if parsed is None: + log.warning("Skipping '%s': could not fetch feed.", show["name"]) + continue + if not is_audio_feed(parsed): + log.info("Skipping '%s' (%s): video podcast, not audio.", show["name"], slug) + skipped_video += 1 + continue + db.execute( + "INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)", + (slug, show["name"], show["feed_url"]), + ) + log.info("Registered new show: %s (%s)", show["name"], slug) + added += 1 db.commit() db.close() + if skipped_video: + log.info("Filtered out %d video podcast(s).", skipped_video) return added def prune_stale_shows(remote_shows): @@ -262,6 +301,10 @@ def fetch_show_episodes(slug, name, feed_url): enclosures = entry.get("enclosures") or [] if not enclosures: continue + # Per-episode audio check (catches mixed-content feeds) + mime = (enclosures[0].get("type") or "").lower() + if mime.startswith("video/"): + continue audio_url = enclosures[0].get("href") if not audio_url: continue @@ -319,6 +362,9 @@ def add_show(feed_url): if parsed is None or not parsed.feed.get("title"): log_error(f"Could not determine show title from {feed_url}") return + if not is_audio_feed(parsed): + log_error(f"Refusing to add '{parsed.feed['title']}': video podcast detected.") + return name = parsed.feed["title"] slug = slugify(name) db = open_subs_db() @@ -364,15 +410,29 @@ def import_opml(path): return shows = parse_opml(content) db = open_subs_db() + added = 0 + skipped_video = 0 for show in shows: slug = slugify(show["name"]) + existing = db.execute("SELECT slug FROM shows WHERE slug = ?", (slug,)).fetchone() + if existing is not None: + continue + parsed = fetch_feed(show["feed_url"]) + if parsed is None: + log.warning("OPML import: skipping '%s', could not fetch feed.", show["name"]) + continue + if not is_audio_feed(parsed): + log.info("OPML import: skipping '%s' (%s): video podcast.", show["name"], slug) + skipped_video += 1 + continue db.execute( - "INSERT OR IGNORE INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)", + "INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)", (slug, show["name"], show["feed_url"]), ) + added += 1 db.commit() db.close() - log.info("OPML import: %d show(s) processed.", len(shows)) + log.info("OPML import: %d added, %d video shows filtered out.", added, skipped_video) def run_fetch(config): g = config["gpodder"] diff --git a/fetch_podcasts.rb b/fetch_podcasts.rb index a9e87b5..c1c8307 100755 --- a/fetch_podcasts.rb +++ b/fetch_podcasts.rb @@ -15,8 +15,8 @@ module RadioAutomation ROOT = File.expand_path("..", __dir__) CONFIG_PATH = File.join(ROOT, "config.json") AUDIO_EXTS = [".mp3", ".m4a"] + VIDEO_EXTS = [".mp4", ".mov", ".avi", ".webm", ".mkv"] - # Resolved at runtime from config.json "storage" key STORAGE_DIR = nil STATE_DIR = nil SUBS_DB = nil @@ -42,6 +42,11 @@ module RadioAutomation append_log("fetch.log", msg) end + def self.log_warning(msg) + puts "#{Time.now.iso8601} [WARN] #{msg}" + append_log("fetch.log", msg) + end + def self.log_error(msg) puts "#{Time.now.iso8601} [ERROR] #{msg}" append_log("fetch.log", msg) @@ -97,6 +102,47 @@ module RadioAutomation db end + # --- Audio-only filtering ------------------------------------------------- + + def self.enclosure_mime_and_url(entry_xml) + enc_m = entry_xml.match(/enclosure[^>]*type="([^"]*)"[^>]*url="([^"]*)"/i) || + entry_xml.match(/enclosure[^>]*url="([^"]*)"[^>]*type="([^"]*)"/i) + if enc_m + if enc_m.pre_match.include?("type=") && enc_m.post_match.empty? + # First pattern matched: group 1 = type, group 2 = url + return [enc_m[1], enc_m[2]] + else + # Second pattern matched: group 1 = url, group 2 = type + return [enc_m[2], enc_m[1]] + end + end + # Fallback: just grab url without type + url_only = entry_xml.match(/enclosure[^>]*url="([^"]*)"/i) + [nil, url_only[1]] if url_only + end + + def self.is_audio_entry?(entry_xml) + mime, url = enclosure_mime_and_url(entry_xml) + return true if mime.nil? && url.nil? # No enclosure; assume ok + mime_l = mime.to_s.downcase + return true if mime_l.start_with?("audio/") + return false if mime_l.start_with?("video/") + # Fall back to URL extension + url_l = url.to_s.downcase + return true if AUDIO_EXTS.any? { |ext| url_l.end_with?(ext) } + return false if VIDEO_EXTS.any? { |ext| url_l.end_with?(ext) } + true # Undetermined: allow + end + + def self.is_audio_feed?(raw_xml) + # Grab the first or block + m = raw_xml.match(/<(?:item|entry)[^>]*>(.*?)<\/(?:item|entry)>/mi) + return true unless m # No items; let through + is_audio_entry?(m[1]) + end + + # --- End audio-only filtering --------------------------------------------- + def self.gpodder_sync(config) g = config["gpodder"] base = g["host"].chomp("/") @@ -161,19 +207,31 @@ module RadioAutomation def self.register_remote_shows(remote_shows) db = open_subs_db added = 0 + skipped_video = 0 remote_shows.each do |show| slug = slugify(show["name"]) existing = db.get_first_value("SELECT slug FROM shows WHERE slug = ?", slug) - if existing.nil? - db.execute( - "INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)", - [slug, show["name"], show["feed_url"]] - ) - log_info("Registered new show: #{show['name']} (#{slug})") - added += 1 + next unless existing.nil? + + raw = fetch_feed(show["feed_url"]) + if raw.nil? + log_warning("Skipping '#{show['name']}': could not fetch feed.") + next end + unless is_audio_feed?(raw) + log_info("Skipping '#{show['name']}' (#{slug}): video podcast, not audio.") + skipped_video += 1 + next + end + db.execute( + "INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)", + [slug, show["name"], show["feed_url"]] + ) + log_info("Registered new show: #{show['name']} (#{slug})") + added += 1 end db.close + log_info("Filtered out #{skipped_video} video podcast(s).") if skipped_video > 0 added end @@ -275,9 +333,13 @@ module RadioAutomation guid = guid_m ? (guid_m[1] || guid_m[2] || guid_m[3]).strip : Digest::MD5.hexdigest(entry_xml[0, 200]) next if seen.include?(guid) - enc_m = entry_xml.match(/enclosure[^>]*url="([^"]+)"/i) - next unless enc_m - audio_url = enc_m[1] + # Per-episode audio check + unless is_audio_entry?(entry_xml) + next + end + + _, audio_url = enclosure_mime_and_url(entry_xml) + next if audio_url.nil? title_m = entry_xml.match(/]*>([^<]*)<\/title>/i) title = title_m ? title_m[1].strip : "untitled" @@ -338,6 +400,10 @@ module RadioAutomation log_error("Could not fetch feed: #{feed_url}") return end + unless is_audio_feed?(raw) + log_error("Refusing to add: video podcast detected at #{feed_url}") + return + end title_m = raw.match(/]*>\s*]*>([^<]*)<\/title>/im) || raw.match(/]*>\s*]*>([^<]*)<\/title>/im) if title_m.nil? @@ -383,15 +449,31 @@ module RadioAutomation content = File.read(path) shows = parse_opml(content) db = open_subs_db + added = 0 + skipped_video = 0 shows.each do |show| slug = slugify(show["name"]) + existing = db.get_first_value("SELECT slug FROM shows WHERE slug = ?", slug) + next unless existing.nil? + + raw = fetch_feed(show["feed_url"]) + if raw.nil? + log_warning("OPML import: skipping '#{show['name']}', could not fetch feed.") + next + end + unless is_audio_feed?(raw) + log_info("OPML import: skipping '#{show['name']}' (#{slug}): video podcast.") + skipped_video += 1 + next + end db.execute( - "INSERT OR IGNORE INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)", + "INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)", [slug, show["name"], show["feed_url"]] ) + added += 1 end db.close - log_info("OPML import: #{shows.size} show(s) processed.") + log_info("OPML import: #{added} added, #{skipped_video} video shows filtered out.") end def self.run_fetch(config)