mirror of
https://github.com/mistergibson/radio.git
synced 2026-09-08 22:09:51 -07:00
audio-only opml filter
This commit is contained in:
parent
ad50821e93
commit
7a57591820
2 changed files with 164 additions and 22 deletions
|
|
@ -5,6 +5,9 @@ for the liquidsoap radio automation stack.
|
|||
|
||||
All data (podcasts, state DBs, logs, playlists) lives under the storage path
|
||||
defined in config.json ("storage" key), keeping the boot drive clean.
|
||||
|
||||
Filters out video podcasts: only shows whose latest enclosure has an
|
||||
audio/* MIME type are registered.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
|
@ -105,6 +108,29 @@ def slugify(name):
|
|||
s = re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_")
|
||||
return s[:60] or "show"
|
||||
|
||||
def is_audio_feed(parsed):
|
||||
"""Check whether the feed's latest episode has an audio enclosure.
|
||||
Returns True if audio, False if video or unknown.
|
||||
"""
|
||||
if not parsed.entries:
|
||||
return True # No entries yet; let it through, will fail on fetch
|
||||
entry = parsed.entries[0]
|
||||
enclosures = entry.get("enclosures") or []
|
||||
if not enclosures:
|
||||
return True # No enclosure info; assume audio
|
||||
mime_type = (enclosures[0].get("type") or "").lower()
|
||||
if mime_type.startswith("audio/"):
|
||||
return True
|
||||
if mime_type.startswith("video/"):
|
||||
return False
|
||||
# Unknown type: check URL extension as fallback
|
||||
url = (enclosures[0].get("href") or "").lower()
|
||||
if any(url.endswith(ext) for ext in AUDIO_EXTS):
|
||||
return True
|
||||
if any(url.endswith(ext) for ext in (".mp4", ".mov", ".avi", ".webm", ".mkv")):
|
||||
return False
|
||||
return True # Default to allowing if undetermined
|
||||
|
||||
def gpodder_sync(cfg):
|
||||
g = cfg["gpodder"]
|
||||
base = g["host"].rstrip("/")
|
||||
|
|
@ -156,18 +182,31 @@ def parse_opml(xml_string):
|
|||
def register_remote_shows(remote_shows):
|
||||
db = open_subs_db()
|
||||
added = 0
|
||||
skipped_video = 0
|
||||
for show in remote_shows:
|
||||
slug = slugify(show["name"])
|
||||
existing = db.execute("SELECT slug FROM shows WHERE slug = ?", (slug,)).fetchone()
|
||||
if existing is None:
|
||||
db.execute(
|
||||
"INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)",
|
||||
(slug, show["name"], show["feed_url"]),
|
||||
)
|
||||
log.info("Registered new show: %s (%s)", show["name"], slug)
|
||||
added += 1
|
||||
if existing is not None:
|
||||
continue
|
||||
# Audio-only filter: check the feed before registering
|
||||
parsed = fetch_feed(show["feed_url"])
|
||||
if parsed is None:
|
||||
log.warning("Skipping '%s': could not fetch feed.", show["name"])
|
||||
continue
|
||||
if not is_audio_feed(parsed):
|
||||
log.info("Skipping '%s' (%s): video podcast, not audio.", show["name"], slug)
|
||||
skipped_video += 1
|
||||
continue
|
||||
db.execute(
|
||||
"INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)",
|
||||
(slug, show["name"], show["feed_url"]),
|
||||
)
|
||||
log.info("Registered new show: %s (%s)", show["name"], slug)
|
||||
added += 1
|
||||
db.commit()
|
||||
db.close()
|
||||
if skipped_video:
|
||||
log.info("Filtered out %d video podcast(s).", skipped_video)
|
||||
return added
|
||||
|
||||
def prune_stale_shows(remote_shows):
|
||||
|
|
@ -262,6 +301,10 @@ def fetch_show_episodes(slug, name, feed_url):
|
|||
enclosures = entry.get("enclosures") or []
|
||||
if not enclosures:
|
||||
continue
|
||||
# Per-episode audio check (catches mixed-content feeds)
|
||||
mime = (enclosures[0].get("type") or "").lower()
|
||||
if mime.startswith("video/"):
|
||||
continue
|
||||
audio_url = enclosures[0].get("href")
|
||||
if not audio_url:
|
||||
continue
|
||||
|
|
@ -319,6 +362,9 @@ def add_show(feed_url):
|
|||
if parsed is None or not parsed.feed.get("title"):
|
||||
log_error(f"Could not determine show title from {feed_url}")
|
||||
return
|
||||
if not is_audio_feed(parsed):
|
||||
log_error(f"Refusing to add '{parsed.feed['title']}': video podcast detected.")
|
||||
return
|
||||
name = parsed.feed["title"]
|
||||
slug = slugify(name)
|
||||
db = open_subs_db()
|
||||
|
|
@ -364,15 +410,29 @@ def import_opml(path):
|
|||
return
|
||||
shows = parse_opml(content)
|
||||
db = open_subs_db()
|
||||
added = 0
|
||||
skipped_video = 0
|
||||
for show in shows:
|
||||
slug = slugify(show["name"])
|
||||
existing = db.execute("SELECT slug FROM shows WHERE slug = ?", (slug,)).fetchone()
|
||||
if existing is not None:
|
||||
continue
|
||||
parsed = fetch_feed(show["feed_url"])
|
||||
if parsed is None:
|
||||
log.warning("OPML import: skipping '%s', could not fetch feed.", show["name"])
|
||||
continue
|
||||
if not is_audio_feed(parsed):
|
||||
log.info("OPML import: skipping '%s' (%s): video podcast.", show["name"], slug)
|
||||
skipped_video += 1
|
||||
continue
|
||||
db.execute(
|
||||
"INSERT OR IGNORE INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)",
|
||||
"INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)",
|
||||
(slug, show["name"], show["feed_url"]),
|
||||
)
|
||||
added += 1
|
||||
db.commit()
|
||||
db.close()
|
||||
log.info("OPML import: %d show(s) processed.", len(shows))
|
||||
log.info("OPML import: %d added, %d video shows filtered out.", added, skipped_video)
|
||||
|
||||
def run_fetch(config):
|
||||
g = config["gpodder"]
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ module RadioAutomation
|
|||
ROOT = File.expand_path("..", __dir__)
|
||||
CONFIG_PATH = File.join(ROOT, "config.json")
|
||||
AUDIO_EXTS = [".mp3", ".m4a"]
|
||||
VIDEO_EXTS = [".mp4", ".mov", ".avi", ".webm", ".mkv"]
|
||||
|
||||
# Resolved at runtime from config.json "storage" key
|
||||
STORAGE_DIR = nil
|
||||
STATE_DIR = nil
|
||||
SUBS_DB = nil
|
||||
|
|
@ -42,6 +42,11 @@ module RadioAutomation
|
|||
append_log("fetch.log", msg)
|
||||
end
|
||||
|
||||
def self.log_warning(msg)
|
||||
puts "#{Time.now.iso8601} [WARN] #{msg}"
|
||||
append_log("fetch.log", msg)
|
||||
end
|
||||
|
||||
def self.log_error(msg)
|
||||
puts "#{Time.now.iso8601} [ERROR] #{msg}"
|
||||
append_log("fetch.log", msg)
|
||||
|
|
@ -97,6 +102,47 @@ module RadioAutomation
|
|||
db
|
||||
end
|
||||
|
||||
# --- Audio-only filtering -------------------------------------------------
|
||||
|
||||
def self.enclosure_mime_and_url(entry_xml)
|
||||
enc_m = entry_xml.match(/enclosure[^>]*type="([^"]*)"[^>]*url="([^"]*)"/i) ||
|
||||
entry_xml.match(/enclosure[^>]*url="([^"]*)"[^>]*type="([^"]*)"/i)
|
||||
if enc_m
|
||||
if enc_m.pre_match.include?("type=") && enc_m.post_match.empty?
|
||||
# First pattern matched: group 1 = type, group 2 = url
|
||||
return [enc_m[1], enc_m[2]]
|
||||
else
|
||||
# Second pattern matched: group 1 = url, group 2 = type
|
||||
return [enc_m[2], enc_m[1]]
|
||||
end
|
||||
end
|
||||
# Fallback: just grab url without type
|
||||
url_only = entry_xml.match(/enclosure[^>]*url="([^"]*)"/i)
|
||||
[nil, url_only[1]] if url_only
|
||||
end
|
||||
|
||||
def self.is_audio_entry?(entry_xml)
|
||||
mime, url = enclosure_mime_and_url(entry_xml)
|
||||
return true if mime.nil? && url.nil? # No enclosure; assume ok
|
||||
mime_l = mime.to_s.downcase
|
||||
return true if mime_l.start_with?("audio/")
|
||||
return false if mime_l.start_with?("video/")
|
||||
# Fall back to URL extension
|
||||
url_l = url.to_s.downcase
|
||||
return true if AUDIO_EXTS.any? { |ext| url_l.end_with?(ext) }
|
||||
return false if VIDEO_EXTS.any? { |ext| url_l.end_with?(ext) }
|
||||
true # Undetermined: allow
|
||||
end
|
||||
|
||||
def self.is_audio_feed?(raw_xml)
|
||||
# Grab the first <item> or <entry> block
|
||||
m = raw_xml.match(/<(?:item|entry)[^>]*>(.*?)<\/(?:item|entry)>/mi)
|
||||
return true unless m # No items; let through
|
||||
is_audio_entry?(m[1])
|
||||
end
|
||||
|
||||
# --- End audio-only filtering ---------------------------------------------
|
||||
|
||||
def self.gpodder_sync(config)
|
||||
g = config["gpodder"]
|
||||
base = g["host"].chomp("/")
|
||||
|
|
@ -161,19 +207,31 @@ module RadioAutomation
|
|||
def self.register_remote_shows(remote_shows)
|
||||
db = open_subs_db
|
||||
added = 0
|
||||
skipped_video = 0
|
||||
remote_shows.each do |show|
|
||||
slug = slugify(show["name"])
|
||||
existing = db.get_first_value("SELECT slug FROM shows WHERE slug = ?", slug)
|
||||
if existing.nil?
|
||||
db.execute(
|
||||
"INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)",
|
||||
[slug, show["name"], show["feed_url"]]
|
||||
)
|
||||
log_info("Registered new show: #{show['name']} (#{slug})")
|
||||
added += 1
|
||||
next unless existing.nil?
|
||||
|
||||
raw = fetch_feed(show["feed_url"])
|
||||
if raw.nil?
|
||||
log_warning("Skipping '#{show['name']}': could not fetch feed.")
|
||||
next
|
||||
end
|
||||
unless is_audio_feed?(raw)
|
||||
log_info("Skipping '#{show['name']}' (#{slug}): video podcast, not audio.")
|
||||
skipped_video += 1
|
||||
next
|
||||
end
|
||||
db.execute(
|
||||
"INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)",
|
||||
[slug, show["name"], show["feed_url"]]
|
||||
)
|
||||
log_info("Registered new show: #{show['name']} (#{slug})")
|
||||
added += 1
|
||||
end
|
||||
db.close
|
||||
log_info("Filtered out #{skipped_video} video podcast(s).") if skipped_video > 0
|
||||
added
|
||||
end
|
||||
|
||||
|
|
@ -275,9 +333,13 @@ module RadioAutomation
|
|||
guid = guid_m ? (guid_m[1] || guid_m[2] || guid_m[3]).strip : Digest::MD5.hexdigest(entry_xml[0, 200])
|
||||
next if seen.include?(guid)
|
||||
|
||||
enc_m = entry_xml.match(/enclosure[^>]*url="([^"]+)"/i)
|
||||
next unless enc_m
|
||||
audio_url = enc_m[1]
|
||||
# Per-episode audio check
|
||||
unless is_audio_entry?(entry_xml)
|
||||
next
|
||||
end
|
||||
|
||||
_, audio_url = enclosure_mime_and_url(entry_xml)
|
||||
next if audio_url.nil?
|
||||
|
||||
title_m = entry_xml.match(/<title[^>]*>([^<]*)<\/title>/i)
|
||||
title = title_m ? title_m[1].strip : "untitled"
|
||||
|
|
@ -338,6 +400,10 @@ module RadioAutomation
|
|||
log_error("Could not fetch feed: #{feed_url}")
|
||||
return
|
||||
end
|
||||
unless is_audio_feed?(raw)
|
||||
log_error("Refusing to add: video podcast detected at #{feed_url}")
|
||||
return
|
||||
end
|
||||
title_m = raw.match(/<channel[^>]*>\s*<title[^>]*>([^<]*)<\/title>/im) ||
|
||||
raw.match(/<feed[^>]*>\s*<title[^>]*>([^<]*)<\/title>/im)
|
||||
if title_m.nil?
|
||||
|
|
@ -383,15 +449,31 @@ module RadioAutomation
|
|||
content = File.read(path)
|
||||
shows = parse_opml(content)
|
||||
db = open_subs_db
|
||||
added = 0
|
||||
skipped_video = 0
|
||||
shows.each do |show|
|
||||
slug = slugify(show["name"])
|
||||
existing = db.get_first_value("SELECT slug FROM shows WHERE slug = ?", slug)
|
||||
next unless existing.nil?
|
||||
|
||||
raw = fetch_feed(show["feed_url"])
|
||||
if raw.nil?
|
||||
log_warning("OPML import: skipping '#{show['name']}', could not fetch feed.")
|
||||
next
|
||||
end
|
||||
unless is_audio_feed?(raw)
|
||||
log_info("OPML import: skipping '#{show['name']}' (#{slug}): video podcast.")
|
||||
skipped_video += 1
|
||||
next
|
||||
end
|
||||
db.execute(
|
||||
"INSERT OR IGNORE INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)",
|
||||
"INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)",
|
||||
[slug, show["name"], show["feed_url"]]
|
||||
)
|
||||
added += 1
|
||||
end
|
||||
db.close
|
||||
log_info("OPML import: #{shows.size} show(s) processed.")
|
||||
log_info("OPML import: #{added} added, #{skipped_video} video shows filtered out.")
|
||||
end
|
||||
|
||||
def self.run_fetch(config)
|
||||
|
|
|
|||
Loading…
Reference in a new issue