From 04098bc1514489d4a3e872f8cd7dc06092c27011 Mon Sep 17 00:00:00 2001 From: "G. Gibson" Date: Tue, 1 Sep 2026 15:27:17 -0700 Subject: [PATCH] Overhaul --- fetch_podcasts.py | 91 +++++++++++++++++-------- fetch_podcasts.rb | 163 ++++++++++++++++++++++++++++++-------------- install_for_jruby | 27 ++++++-- install_for_python | 21 +++++- update_playlists.py | 117 ++++++++++++++++++------------- update_playlists.rb | 153 +++++++++++++++++++++++++++-------------- 6 files changed, 385 insertions(+), 187 deletions(-) diff --git a/fetch_podcasts.py b/fetch_podcasts.py index 18c6b01..8f472da 100755 --- a/fetch_podcasts.py +++ b/fetch_podcasts.py @@ -8,6 +8,9 @@ defined in config.json ("storage" key), keeping the boot drive clean. Filters out video podcasts: only shows whose latest enclosure has an audio/* MIME type are registered. + +Archived shows (archived=1) have episodes downloaded to disk; non-archived +shows (archived=0) store only the enclosure URL for live streaming. """ import argparse @@ -17,6 +20,7 @@ import re import shutil import sqlite3 import sys +import uuid import xml.etree.ElementTree as ET from pathlib import Path from urllib.parse import quote @@ -28,6 +32,7 @@ ROOT = Path(__file__).resolve().parent CONFIG_PATH = ROOT / "config.json" AUDIO_EXTS = {".mp3", ".m4a"} +VIDEO_EXTS = {".mp4", ".mov", ".avi", ".webm", ".mkv"} STATE_DIR = None SUBS_DB = None @@ -76,10 +81,12 @@ def open_subs_db(): conn.execute(""" CREATE TABLE IF NOT EXISTS shows ( slug TEXT PRIMARY KEY, + guid TEXT NOT NULL UNIQUE, name TEXT NOT NULL, feed_url TEXT NOT NULL UNIQUE, source TEXT DEFAULT 'manual', opml_import INTEGER DEFAULT 0, + archived INTEGER DEFAULT 1, created_at TEXT DEFAULT (datetime('now')) ) """) @@ -96,8 +103,10 @@ def open_played_db(): guid TEXT NOT NULL, title TEXT, file_path TEXT, - duration_seconds INTEGER, - played_at TEXT DEFAULT (datetime('now')), + enclosure_url TEXT, + runlength INTEGER, + played INTEGER DEFAULT 0, + played_at TEXT, UNIQUE(show_slug, guid) ) """) @@ -108,6 +117,9 @@ def slugify(name): s = re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_") return s[:60] or "show" +def gen_uuid(): + return str(uuid.uuid4()) + def is_audio_feed(parsed): """Check whether the feed's latest episode has an audio enclosure. Returns True if audio, False if video or unknown. @@ -123,13 +135,12 @@ def is_audio_feed(parsed): return True if mime_type.startswith("video/"): return False - # Unknown type: check URL extension as fallback url = (enclosures[0].get("href") or "").lower() if any(url.endswith(ext) for ext in AUDIO_EXTS): return True - if any(url.endswith(ext) for ext in (".mp4", ".mov", ".avi", ".webm", ".mkv")): + if any(url.endswith(ext) for ext in VIDEO_EXTS): return False - return True # Default to allowing if undetermined + return True def gpodder_sync(cfg): g = cfg["gpodder"] @@ -174,9 +185,14 @@ def parse_opml(xml_string): for outline in root.iter("outline"): feed_url = (outline.attrib.get("xmlUrl") or "").strip() name = (outline.attrib.get("text") or "").strip() + guid = (outline.attrib.get("guid") or "").strip() if not feed_url or not re.match(r"^https?://", feed_url): continue - shows.append({"name": name, "feed_url": feed_url}) + shows.append({ + "name": name, + "feed_url": feed_url, + "guid": guid if guid else None, + }) return shows def register_remote_shows(remote_shows): @@ -188,7 +204,6 @@ def register_remote_shows(remote_shows): existing = db.execute("SELECT slug FROM shows WHERE slug = ?", (slug,)).fetchone() if existing is not None: continue - # Audio-only filter: check the feed before registering parsed = fetch_feed(show["feed_url"]) if parsed is None: log.warning("Skipping '%s': could not fetch feed.", show["name"]) @@ -197,9 +212,10 @@ def register_remote_shows(remote_shows): log.info("Skipping '%s' (%s): video podcast, not audio.", show["name"], slug) skipped_video += 1 continue + guid = show["guid"] or gen_uuid() db.execute( - "INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)", - (slug, show["name"], show["feed_url"]), + "INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'gpodder', 0, 1)", + (slug, guid, show["name"], show["feed_url"]), ) log.info("Registered new show: %s (%s)", show["name"], slug) added += 1 @@ -281,6 +297,12 @@ def safe_filename(title, fallback): name = re.sub(r"[^\w\s.-]", "", title or "").strip().replace(" ", "_") return (name[:120] or fallback) + ".mp3" +def show_archived(subs_db, slug): + row = subs_db.execute("SELECT archived FROM shows WHERE slug = ?", (slug,)).fetchone() + if row is None: + return True + return row["archived"] == 1 + def fetch_show_episodes(slug, name, feed_url): dest_dir = PODCASTS_DIR / slug dest_dir.mkdir(parents=True, exist_ok=True) @@ -293,6 +315,7 @@ def fetch_show_episodes(slug, name, feed_url): for row in played_db.execute("SELECT guid FROM episodes WHERE show_slug = ?", (slug,)) } subs_db = open_subs_db() + archived = show_archived(subs_db, slug) new_count = 0 for entry in parsed.entries: guid = entry.get("id") or entry.get("link") or entry.get("title", "") @@ -301,7 +324,6 @@ def fetch_show_episodes(slug, name, feed_url): enclosures = entry.get("enclosures") or [] if not enclosures: continue - # Per-episode audio check (catches mixed-content feeds) mime = (enclosures[0].get("type") or "").lower() if mime.startswith("video/"): continue @@ -309,18 +331,27 @@ def fetch_show_episodes(slug, name, feed_url): if not audio_url: continue title = entry.get("title", "untitled") - filename = safe_filename(title, guid[-20:]) - file_path = download_episode(audio_url, dest_dir, filename) - if file_path is None: - continue duration = extract_duration(entry) - played_db.execute( - "INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, duration_seconds, played_at) " - "VALUES (?, ?, ?, ?, ?, NULL)", - (slug, guid, title, file_path, duration), - ) + + if archived: + filename = safe_filename(title, guid[-20:]) + file_path = download_episode(audio_url, dest_dir, filename) + if file_path is None: + continue + played_db.execute( + "INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, enclosure_url, runlength, played) " + "VALUES (?, ?, ?, ?, ?, ?, 0)", + (slug, guid, title, file_path, audio_url, duration), + ) + else: + played_db.execute( + "INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, enclosure_url, runlength, played) " + "VALUES (?, ?, ?, NULL, ?, ?, 0)", + (slug, guid, title, audio_url, duration), + ) new_count += 1 - log.info(" New episode: %s [%s]", title, filename) + kind = "downloaded" if archived else "live" + log.info(" New episode: %s [%s]", title, kind) played_db.commit() played_db.close() subs_db.close() @@ -343,16 +374,16 @@ def fetch_all_episodes(): def list_shows(detail=False): db = open_subs_db() rows = db.execute( - "SELECT slug, name, feed_url, source, opml_import FROM shows ORDER BY name" + "SELECT slug, name, feed_url, source, opml_import, archived FROM shows ORDER BY name" ).fetchall() db.close() if not rows: print("No shows registered.") return - print(f"{'SLUG':<30} {'PROTECTED':<10} NAME") + print(f"{'SLUG':<30} {'ARCHIVED':<10} {'SOURCE':<10} NAME") for r in rows: - prot = "yes" if r["opml_import"] else "no" - line = f"{r['slug']:<30} {prot:<10} {r['name']}" + arch = "yes" if r["archived"] == 1 else "no" + line = f"{r['slug']:<30} {arch:<10} {r['source']:<10} {r['name']}" if detail: line += f"\n{'':<50} {r['feed_url']}" print(line) @@ -367,10 +398,11 @@ def add_show(feed_url): return name = parsed.feed["title"] slug = slugify(name) + guid = gen_uuid() db = open_subs_db() db.execute( - "INSERT OR IGNORE INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'manual', 0)", - (slug, name, feed_url), + "INSERT OR IGNORE INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'manual', 0, 1)", + (slug, guid, name, feed_url), ) db.commit() db.close() @@ -381,7 +413,7 @@ def remove_show_data(slug): pod_dir = PODCASTS_DIR / slug if pod_dir.exists(): shutil.rmtree(pod_dir) - pls = PLAYLISTS_DIR / f"{slug}.pls" + pls = PLAYLISTS_DIR / f"{slug}.txt" if pls.exists(): pls.unlink() @@ -425,9 +457,10 @@ def import_opml(path): log.info("OPML import: skipping '%s' (%s): video podcast.", show["name"], slug) skipped_video += 1 continue + guid = show["guid"] or gen_uuid() db.execute( - "INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)", - (slug, show["name"], show["feed_url"]), + "INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'opml', 1, 1)", + (slug, guid, show["name"], show["feed_url"]), ) added += 1 db.commit() diff --git a/fetch_podcasts.rb b/fetch_podcasts.rb index c1c8307..ad23081 100755 --- a/fetch_podcasts.rb +++ b/fetch_podcasts.rb @@ -5,12 +5,15 @@ require "net/http" require "uri" require "cgi" require "json" -require "sqlite3" +require "jdbc/sqlite3" require "rexml/document" require "digest/md5" +require "securerandom" require "fileutils" require "optparse" +Jdbc::SQLite3.load_driver + module RadioAutomation ROOT = File.expand_path("..", __dir__) CONFIG_PATH = File.join(ROOT, "config.json") @@ -37,6 +40,37 @@ module RadioAutomation self.PLAYLISTS_DIR = File.join(@storage, "playlists") end + # --- JDBC connection helpers --------------------------------------------- + + def self.jdb_connect(db_file) + java.sql.DriverManager.getConnection("jdbc:sqlite:#{db_file}") + end + + def self.jdb_query(conn, sql, params = []) + stmt = conn.prepareStatement(sql) + params.each_with_index { |p, i| stmt.setObject(i + 1, p) } + rs = stmt.executeQuery + cols = [] + meta = rs.getMetaData + (1..meta.getColumnCount).each { |i| cols << meta.getColumnName(i) } + rows = [] + while rs.next + row = {} + cols.each { |c| row[c] = rs.getObject(c) } + rows << row + end + rs.close + stmt.close + rows + end + + def self.jdb_exec(conn, sql, params = []) + stmt = conn.prepareStatement(sql) + params.each_with_index { |p, i| stmt.setObject(i + 1, p) } + stmt.executeUpdate + stmt.close + end + def self.log_info(msg) puts "#{Time.now.iso8601} [INFO] #{msg}" append_log("fetch.log", msg) @@ -68,16 +102,21 @@ module RadioAutomation s[0, 60] || "show" end + def self.gen_uuid + SecureRandom.uuid + end + def self.open_subs_db - db = SQLite3::Database.new(SUBS_DB) - db.results_as_hash = true - db.execute <<-SQL + db = jdb_connect(SUBS_DB) + jdb_exec(db, <<-SQL) CREATE TABLE IF NOT EXISTS shows ( slug TEXT PRIMARY KEY, + guid TEXT NOT NULL UNIQUE, name TEXT NOT NULL, feed_url TEXT NOT NULL UNIQUE, source TEXT DEFAULT 'manual', opml_import INTEGER DEFAULT 0, + archived INTEGER DEFAULT 1, created_at TEXT DEFAULT (datetime('now')) ) SQL @@ -85,16 +124,17 @@ module RadioAutomation end def self.open_played_db - db = SQLite3::Database.new(PLAYED_DB) - db.results_as_hash = true - db.execute <<-SQL + db = jdb_connect(PLAYED_DB) + jdb_exec(db, <<-SQL) CREATE TABLE IF NOT EXISTS episodes ( id INTEGER PRIMARY KEY AUTOINCREMENT, show_slug TEXT NOT NULL, guid TEXT NOT NULL, title TEXT, file_path TEXT, - duration_seconds INTEGER, + enclosure_url TEXT, + runlength INTEGER, + played INTEGER DEFAULT 0, played_at TEXT, UNIQUE(show_slug, guid) ) @@ -109,35 +149,30 @@ module RadioAutomation entry_xml.match(/enclosure[^>]*url="([^"]*)"[^>]*type="([^"]*)"/i) if enc_m if enc_m.pre_match.include?("type=") && enc_m.post_match.empty? - # First pattern matched: group 1 = type, group 2 = url return [enc_m[1], enc_m[2]] else - # Second pattern matched: group 1 = url, group 2 = type return [enc_m[2], enc_m[1]] end end - # Fallback: just grab url without type url_only = entry_xml.match(/enclosure[^>]*url="([^"]*)"/i) [nil, url_only[1]] if url_only end def self.is_audio_entry?(entry_xml) mime, url = enclosure_mime_and_url(entry_xml) - return true if mime.nil? && url.nil? # No enclosure; assume ok + return true if mime.nil? && url.nil? mime_l = mime.to_s.downcase return true if mime_l.start_with?("audio/") return false if mime_l.start_with?("video/") - # Fall back to URL extension url_l = url.to_s.downcase return true if AUDIO_EXTS.any? { |ext| url_l.end_with?(ext) } return false if VIDEO_EXTS.any? { |ext| url_l.end_with?(ext) } - true # Undetermined: allow + true end def self.is_audio_feed?(raw_xml) - # Grab the first or block m = raw_xml.match(/<(?:item|entry)[^>]*>(.*?)<\/(?:item|entry)>/mi) - return true unless m # No items; let through + return true unless m is_audio_entry?(m[1]) end @@ -195,8 +230,9 @@ module RadioAutomation REXML::XPath.each(doc, "//outline[@xmlUrl]") do |node| feed_url = node.attributes["xmlUrl"].to_s.strip name = node.attributes["text"].to_s.strip + guid = node.attributes["guid"].to_s.strip next unless feed_url =~ /\Ahttps?:\/\// - shows << { "name" => name, "feed_url" => feed_url } + shows << { "name" => name, "feed_url" => feed_url, "guid" => (guid.empty? ? nil : guid) } end shows rescue REXML::ParseException => e @@ -210,8 +246,8 @@ module RadioAutomation skipped_video = 0 remote_shows.each do |show| slug = slugify(show["name"]) - existing = db.get_first_value("SELECT slug FROM shows WHERE slug = ?", slug) - next unless existing.nil? + existing = jdb_query(db, "SELECT slug FROM shows WHERE slug = ?", [slug]) + next unless existing.empty? raw = fetch_feed(show["feed_url"]) if raw.nil? @@ -223,9 +259,11 @@ module RadioAutomation skipped_video += 1 next end - db.execute( - "INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)", - [slug, show["name"], show["feed_url"]] + guid = show["guid"] || gen_uuid + jdb_exec( + db, + "INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'gpodder', 0, 1)", + [slug, guid, show["name"], show["feed_url"]] ) log_info("Registered new show: #{show['name']} (#{slug})") added += 1 @@ -238,12 +276,12 @@ module RadioAutomation def self.prune_stale_shows(remote_shows) db = open_subs_db remote_slugs = remote_shows.map { |s| slugify(s["name"]) } - stale = db.query_all("SELECT slug, name FROM shows WHERE source = 'gpodder' AND opml_import = 0") + stale = jdb_query(db, "SELECT slug, name FROM shows WHERE source = 'gpodder' AND opml_import = 0") removed = 0 stale.each do |row| next if remote_slugs.include?(row["slug"]) remove_show_data(row["slug"]) - db.execute("DELETE FROM shows WHERE slug = ?", [row["slug"]]) + jdb_exec(db, "DELETE FROM shows WHERE slug = ?", [row["slug"]]) log_info("Pruned stale show: #{row['name']} (#{row['slug']})") removed += 1 end @@ -318,14 +356,21 @@ module RadioAutomation "#{name || fallback}.mp3" end + def self.show_archived?(subs_db, slug) + rows = jdb_query(subs_db, "SELECT archived FROM shows WHERE slug = ?", [slug]) + rows.empty? ? true : (rows.first["archived"] == 1) + end + def self.fetch_show_episodes(slug, name, feed_url) dest_dir = File.join(PODCASTS_DIR, slug) raw = fetch_feed(feed_url) return 0 if raw.nil? played_db = open_played_db - seen = played_db.query("SELECT guid FROM episodes WHERE show_slug = ?", slug).map { |r| r["guid"] } + seen_rows = jdb_query(played_db, "SELECT guid FROM episodes WHERE show_slug = ?", [slug]) + seen = seen_rows.map { |r| r["guid"] } subs_db = open_subs_db + archived = show_archived?(subs_db, slug) new_count = 0 raw.scan(/<(?:item|entry)[^>]*>(.*?)<\/(?:item|entry)>/mi).each do |(entry_xml)| @@ -333,7 +378,6 @@ module RadioAutomation guid = guid_m ? (guid_m[1] || guid_m[2] || guid_m[3]).strip : Digest::MD5.hexdigest(entry_xml[0, 200]) next if seen.include?(guid) - # Per-episode audio check unless is_audio_entry?(entry_xml) next end @@ -343,17 +387,26 @@ module RadioAutomation title_m = entry_xml.match(/]*>([^<]*)<\/title>/i) title = title_m ? title_m[1].strip : "untitled" - filename = safe_filename(title, guid[-20..]) - file_path = download_episode(audio_url, dest_dir, filename) - next if file_path.nil? - duration = extract_duration(entry_xml) - played_db.execute( - "INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, duration_seconds, played_at) VALUES (?, ?, ?, ?, ?, NULL)", - [slug, guid, title, file_path, duration] - ) + + if archived + filename = safe_filename(title, guid[-20..]) + file_path = download_episode(audio_url, dest_dir, filename) + next if file_path.nil? + jdb_exec( + played_db, + "INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, enclosure_url, runlength, played) VALUES (?, ?, ?, ?, ?, ?, 0)", + [slug, guid, title, file_path, audio_url, duration] + ) + else + jdb_exec( + played_db, + "INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, enclosure_url, runlength, played) VALUES (?, ?, ?, NULL, ?, ?, 0)", + [slug, guid, title, audio_url, duration] + ) + end new_count += 1 - log_info(" New episode: #{title} [#{filename}]") + log_info(" New episode: #{title} [#{archived ? 'downloaded' : 'live'}]") end played_db.close @@ -363,7 +416,7 @@ module RadioAutomation def self.fetch_all_episodes db = open_subs_db - shows = db.query_all("SELECT slug, name, feed_url FROM shows ORDER BY name") + shows = jdb_query(db, "SELECT slug, name, feed_url FROM shows ORDER BY name") db.close total_new = 0 shows.each do |show| @@ -379,16 +432,16 @@ module RadioAutomation def self.list_shows(detail: false) db = open_subs_db - rows = db.query_all("SELECT slug, name, feed_url, source, opml_import FROM shows ORDER BY name") + rows = jdb_query(db, "SELECT slug, name, feed_url, source, opml_import, archived FROM shows ORDER BY name") db.close if rows.empty? puts "No shows registered." return end - puts format("%-30s %-10s %s", "SLUG", "PROTECTED", "NAME") + puts format("%-30s %-10s %-8s %s", "SLUG", "ARCHIVED", "SOURCE", "NAME") rows.each do |r| - prot = r["opml_import"] ? "yes" : "no" - line = format("%-30s %-10s %s", r["slug"], prot, r["name"]) + arch = r["archived"] == 1 ? "yes" : "no" + line = format("%-30s %-10s %-8s %s", r["slug"], arch, r["source"], r["name"]) line += "\n" + (" " * 50) + r["feed_url"] if detail puts line end @@ -412,10 +465,12 @@ module RadioAutomation end name = title_m[1].strip slug = slugify(name) + guid = gen_uuid db = open_subs_db - db.execute( - "INSERT OR IGNORE INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'manual', 0)", - [slug, name, feed_url] + jdb_exec( + db, + "INSERT OR IGNORE INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'manual', 0, 1)", + [slug, guid, name, feed_url] ) db.close log_info("Added show: #{name} (#{slug})") @@ -431,18 +486,18 @@ module RadioAutomation def self.delete_show(slug) db = open_subs_db - row = db.get_first_hash("SELECT name FROM shows WHERE slug = ?", slug) - if row.nil? + rows = jdb_query(db, "SELECT name FROM shows WHERE slug = ?", [slug]) + if rows.empty? log_error("No show found with slug '#{slug}'.") return end remove_show_data(slug) - db.execute("DELETE FROM shows WHERE slug = ?", [slug]) + jdb_exec(db, "DELETE FROM shows WHERE slug = ?", [slug]) db.close played_db = open_played_db - played_db.execute("DELETE FROM episodes WHERE show_slug = ?", [slug]) + jdb_exec(played_db, "DELETE FROM episodes WHERE show_slug = ?", [slug]) played_db.close - log_info("Deleted show: #{row['name']} (#{slug})") + log_info("Deleted show: #{rows.first['name']} (#{slug})") end def self.import_opml(path) @@ -453,8 +508,8 @@ module RadioAutomation skipped_video = 0 shows.each do |show| slug = slugify(show["name"]) - existing = db.get_first_value("SELECT slug FROM shows WHERE slug = ?", slug) - next unless existing.nil? + existing = jdb_query(db, "SELECT slug FROM shows WHERE slug = ?", [slug]) + next unless existing.empty? raw = fetch_feed(show["feed_url"]) if raw.nil? @@ -466,9 +521,11 @@ module RadioAutomation skipped_video += 1 next end - db.execute( - "INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)", - [slug, show["name"], show["feed_url"]] + guid = show["guid"] || gen_uuid + jdb_exec( + db, + "INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'opml', 1, 1)", + [slug, guid, show["name"], show["feed_url"]] ) added += 1 end diff --git a/install_for_jruby b/install_for_jruby index 85ab5a9..7a7336a 100755 --- a/install_for_jruby +++ b/install_for_jruby @@ -21,12 +21,27 @@ echo "Install dir: ${SCRIPT_DIR}" echo "Using jruby: ${JRuby_BIN}" echo "" +# --- Verify required source files exist ------------------------------------ +for req in station.liq fetch_podcasts.rb update_playlists.rb; do + if [ ! -f "${SCRIPT_DIR}/${req}" ]; then + echo "ERROR: required file '${req}' not found in ${SCRIPT_DIR}" >&2 + exit 1 + fi +done + # --- Prompt for storage path --------------------------------------------- DEFAULT_STORAGE="/srv/radio-storage" read -rp "Enter storage path for media/state/logs [${DEFAULT_STORAGE}]: " STORAGE_PATH STORAGE_PATH="${STORAGE_PATH:-$DEFAULT_STORAGE}" mkdir -p "${STORAGE_PATH}"/{music,podcasts,jingles,announcements,state,playlists,logs} -chown -R liquidsoap:liquidsoap "${STORAGE_PATH}" 2>/dev/null || true +# Media dirs are read by the liquidsoap user at stream time +chown -R liquidsoap:liquidsoap "${STORAGE_PATH}"/{music,podcasts,jingles,announcements} 2>/dev/null || true +# state/ and playlists/ are written by the cron jobs (run as root) and read by liquidsoap +chown root:liquidsoap "${STORAGE_PATH}"/{state,playlists} 2>/dev/null || true +chmod 775 "${STORAGE_PATH}"/{state,playlists} 2>/dev/null || true +# logs/ is appended to by cron +chown root:liquidsoap "${STORAGE_PATH}/logs" 2>/dev/null || true +chmod 775 "${STORAGE_PATH}/logs" 2>/dev/null || true # --- Collect Icecast credentials ------------------------------------------ read -rp "Icecast host [127.0.0.1]: " IC_HOST @@ -78,10 +93,12 @@ chmod 600 "$CONFIG_FILE" echo "Wrote ${CONFIG_FILE}" # --- Gem check (install one at a time to avoid memory limits) -------------- -for gem in sqlite3 json; do +# Use jdbc-sqlite3, not sqlite3: the latter needs a native C extension that +# fails to build on JRuby. jdbc-sqlite3 ships the SQLite JDBC driver as a JAR. +for gem in jdbc-sqlite3 json; do if ! "${JRuby_BIN}" -e "require '${gem}'" >/dev/null 2>&1; then echo "Installing gem: ${gem}" - gem install "${gem}" + "${JRuby_BIN}" -S gem install "${gem}" fi done @@ -109,7 +126,9 @@ echo "Enabled systemd unit: ${SERVICE_NAME}" CRON_FETCH="0 * * * * cd ${SCRIPT_DIR} && ${JRuby_BIN} fetch_podcasts.rb >> ${STORAGE_PATH}/logs/cron.log 2>&1" CRON_PLAYLISTS="30 * * * * cd ${SCRIPT_DIR} && ${JRuby_BIN} update_playlists.rb >> ${STORAGE_PATH}/logs/cron.log 2>&1" -(crontab -l 2>/dev/null | grep -v "fetch_podcasts.rb\|update_playlists.rb"; echo "$CRON_FETCH"; echo "$CRON_PLAYLISTS") | crontab - +# Guard against pipefail killing us when crontab is empty or has no matching lines +EXISTING_CRON=$(crontab -l 2>/dev/null | grep -v "fetch_podcasts.rb\|update_playlists.rb" || true) +( [ -n "$EXISTING_CRON" ] && echo "$EXISTING_CRON"; echo "$CRON_FETCH"; echo "$CRON_PLAYLISTS" ) | crontab - echo "Installed cron jobs:" echo " $CRON_FETCH" echo " $CRON_PLAYLISTS" diff --git a/install_for_python b/install_for_python index 152b8a5..31665c6 100755 --- a/install_for_python +++ b/install_for_python @@ -20,12 +20,27 @@ echo "=== ${SERVICE_NAME} installer (Python) ===" echo "Install dir: ${SCRIPT_DIR}" echo "" +# --- Verify required source files exist ------------------------------------ +for req in station.liq fetch_podcasts.py update_playlists.py; do + if [ ! -f "${SCRIPT_DIR}/${req}" ]; then + echo "ERROR: required file '${req}' not found in ${SCRIPT_DIR}" >&2 + exit 1 + fi +done + # --- Prompt for storage path --------------------------------------------- DEFAULT_STORAGE="/srv/radio-storage" read -rp "Enter storage path for media/state/logs [${DEFAULT_STORAGE}]: " STORAGE_PATH STORAGE_PATH="${STORAGE_PATH:-$DEFAULT_STORAGE}" mkdir -p "${STORAGE_PATH}"/{music,podcasts,jingles,announcements,state,playlists,logs} -chown -R liquidsoap:liquidsoap "${STORAGE_PATH}" 2>/dev/null || true +# Media dirs are read by the liquidsoap user at stream time +chown -R liquidsoap:liquidsoap "${STORAGE_PATH}"/{music,podcasts,jingles,announcements} 2>/dev/null || true +# state/ and playlists/ are written by the cron jobs (run as root) and read by liquidsoap +chown root:liquidsoap "${STORAGE_PATH}"/{state,playlists} 2>/dev/null || true +chmod 775 "${STORAGE_PATH}"/{state,playlists} 2>/dev/null || true +# logs/ is appended to by cron +chown root:liquidsoap "${STORAGE_PATH}/logs" 2>/dev/null || true +chmod 775 "${STORAGE_PATH}/logs" 2>/dev/null || true # --- Collect Icecast credentials ------------------------------------------ read -rp "Icecast host [127.0.0.1]: " IC_HOST @@ -111,7 +126,9 @@ echo "Enabled systemd unit: ${SERVICE_NAME}" CRON_FETCH="0 * * * * cd ${SCRIPT_DIR} && ${PYBIN} fetch_podcasts.py >> ${STORAGE_PATH}/logs/cron.log 2>&1" CRON_PLAYLISTS="30 * * * * cd ${SCRIPT_DIR} && ${PYBIN} update_playlists.py >> ${STORAGE_PATH}/logs/cron.log 2>&1" -(crontab -l 2>/dev/null | grep -v "fetch_podcasts.py\|update_playlists.py"; echo "$CRON_FETCH"; echo "$CRON_PLAYLISTS") | crontab - +# Guard against pipefail killing us when crontab is empty or has no matching lines +EXISTING_CRON=$(crontab -l 2>/dev/null | grep -v "fetch_podcasts.py\|update_playlists.py" || true) +( [ -n "$EXISTING_CRON" ] && echo "$EXISTING_CRON"; echo "$CRON_FETCH"; echo "$CRON_PLAYLISTS" ) | crontab - echo "Installed cron jobs:" echo " $CRON_FETCH" echo " $CRON_PLAYLISTS" diff --git a/update_playlists.py b/update_playlists.py index 7294116..545217d 100755 --- a/update_playlists.py +++ b/update_playlists.py @@ -1,7 +1,12 @@ #!/usr/bin/env python3 """ -update_playlists.py - Regenerate per-show .pls playlists based on playback history. -Runs via cron hourly at :30. All data under the storage path from config.json. +update_playlists.py - Select the next unplayed episode per show and write an +annotated URI line for station.liq to consume. Runs via cron hourly at :30. + +Selection is driven by the episodes table (keyed on guid + played flag), not +by scanning the filesystem, so it works identically for archived shows +(local file_path) and live shows (remote enclosure_url). All data under the +storage path from config.json. """ import argparse @@ -14,8 +19,6 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent CONFIG_PATH = ROOT / "config.json" -AUDIO_EXTS = {".mp3", ".m4a"} - STATE_DIR = None SUBS_DB = None PLAYED_DB = None @@ -55,6 +58,19 @@ def _setup_logging(): def open_subs_db(): conn = sqlite3.connect(SUBS_DB) conn.row_factory = sqlite3.Row + conn.execute(""" + CREATE TABLE IF NOT EXISTS shows ( + slug TEXT PRIMARY KEY, + guid TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + feed_url TEXT NOT NULL UNIQUE, + source TEXT DEFAULT 'manual', + opml_import INTEGER DEFAULT 0, + archived INTEGER DEFAULT 1, + created_at TEXT DEFAULT (datetime('now')) + ) + """) + conn.commit() return conn def open_played_db(): @@ -67,7 +83,9 @@ def open_played_db(): guid TEXT NOT NULL, title TEXT, file_path TEXT, - duration_seconds INTEGER, + enclosure_url TEXT, + runlength INTEGER, + played INTEGER DEFAULT 0, played_at TEXT, UNIQUE(show_slug, guid) ) @@ -75,60 +93,64 @@ def open_played_db(): conn.commit() return conn -def find_audio_files(directory): - results = [] - if not directory.exists(): - return results - for p in sorted(directory.rglob("*")): - if p.is_file() and p.suffix.lower() in AUDIO_EXTS: - results.append(str(p)) - return results - def select_unplayed_episode(slug, played_db): - files = find_audio_files(PODCASTS_DIR / slug) - if not files: - return None - played_rows = played_db.execute( - "SELECT file_path, played_at FROM episodes WHERE show_slug = ? AND played_at IS NOT NULL", + """Pick the next unplayed episode for a show, keyed by guid. + Archived -> local file_path; Live -> remote enclosure_url.""" + row = played_db.execute( + "SELECT guid, title, file_path, enclosure_url, runlength " + "FROM episodes WHERE show_slug = ? AND played = 0 ORDER BY id ASC LIMIT 1", (slug,), - ).fetchall() - played_paths = {row["file_path"]: row["played_at"] for row in played_rows} - unplayed = [f for f in files if f not in played_paths] - if unplayed: - return unplayed[0] - if played_paths: - return min(played_paths.items(), key=lambda kv: (kv[1] or ""))[0] - return files[0] + ).fetchone() + return row -def write_pls(filepath, out_path): - abs_path = str(Path(filepath).resolve()) - content = f"[playlist]\nFile1={abs_path}\nTitle1=Radio Episode\nLength1=-1\nNumberOfEntries=1\nVersion=2\n" +def annotated_uri(ep): + """Build the annotate: URI line station.liq consumes. + Archived -> local file path; Live -> remote enclosure URL.""" + uri = ep["file_path"] or ep["enclosure_url"] + if not uri: + return None + rl = int(ep["runlength"] or 0) + title = (ep["title"] or "").replace('"', "'") + return f'annotate:liq_runlength="{rl}",liq_title="{title}":{uri}' + +def write_queue_line(slug, line, out_path): out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(content) + out_path.write_text(line + "\n") -def mark_as_played(slug, filepath, played_db): +def mark_as_played(slug, guid, played_db): played_db.execute( - "UPDATE episodes SET played_at = datetime('now') WHERE show_slug = ? AND file_path = ?", - (slug, filepath), + "UPDATE episodes SET played = 1, played_at = datetime('now') WHERE show_slug = ? AND guid = ?", + (slug, guid), ) played_db.commit() def update_all(): subs_db = open_subs_db() played_db = open_played_db() - shows = subs_db.execute("SELECT slug, name FROM shows ORDER BY name").fetchall() + shows = subs_db.execute("SELECT slug, name, archived FROM shows ORDER BY name").fetchall() + queued = 0 + skipped = 0 for show in shows: slug = show["slug"] - selected = select_unplayed_episode(slug, played_db) - if selected is None: - log.info("%s: no audio files found, skipping.", slug) + ep = select_unplayed_episode(slug, played_db) + if ep is None: + log.info("%s: no unplayed episodes, skipping.", slug) + skipped += 1 continue - out_pls = PLAYLISTS_DIR / f"{slug}.pls" - write_pls(selected, out_pls) - mark_as_played(slug, selected, played_db) - log.info("%s: queued %s", slug, Path(selected).name) + line = annotated_uri(ep) + if line is None: + log.info("%s: episode %s has no usable URI, skipping.", slug, ep["guid"]) + skipped += 1 + continue + out_txt = PLAYLISTS_DIR / f"{slug}.txt" + write_queue_line(slug, line, out_txt) + mark_as_played(slug, ep["guid"], played_db) + kind = "downloaded" if ep["file_path"] else "live" + log.info("%s: queued %s [%s] runlength=%ss", slug, ep["title"], kind, int(ep["runlength"] or 0)) + queued += 1 subs_db.close() played_db.close() + log.info("=== Update complete: %d queued, %d skipped ===", queued, skipped) def json_summary(): subs_db = open_subs_db() @@ -137,12 +159,13 @@ def json_summary(): summary = {} for show in shows: slug = show["slug"] - files = find_audio_files(PODCASTS_DIR / slug) - played_count = played_db.execute( - "SELECT COUNT(*) as c FROM episodes WHERE show_slug = ? AND played_at IS NOT NULL", - (slug,), + total = played_db.execute( + "SELECT COUNT(*) as c FROM episodes WHERE show_slug = ?", (slug,) ).fetchone()["c"] - summary[slug] = {"total_files": len(files), "played_count": played_count} + played = played_db.execute( + "SELECT COUNT(*) as c FROM episodes WHERE show_slug = ? AND played = 1", (slug,) + ).fetchone()["c"] + summary[slug] = {"total_episodes": total, "played_count": played, "unplayed": total - played} subs_db.close() played_db.close() print(json.dumps(summary, indent=2)) diff --git a/update_playlists.rb b/update_playlists.rb index a7e1dcf..e628050 100755 --- a/update_playlists.rb +++ b/update_playlists.rb @@ -2,14 +2,15 @@ # frozen_string_literal: true require "json" -require "sqlite3" +require "jdbc/sqlite3" require "fileutils" require "optparse" +Jdbc::SQLite3.load_driver + module RadioAutomation ROOT = File.expand_path("..", __dir__) CONFIG_PATH = File.join(ROOT, "config.json") - AUDIO_EXTS = [".mp3", ".m4a"] STORAGE_DIR = nil STATE_DIR = nil @@ -31,6 +32,37 @@ module RadioAutomation self.LOGS_DIR = File.join(@storage, "logs") end + # --- JDBC connection helpers --------------------------------------------- + + def self.jdb_connect(db_file) + java.sql.DriverManager.getConnection("jdbc:sqlite:#{db_file}") + end + + def self.jdb_query(conn, sql, params = []) + stmt = conn.prepareStatement(sql) + params.each_with_index { |p, i| stmt.setObject(i + 1, p) } + rs = stmt.executeQuery + cols = [] + meta = rs.getMetaData + (1..meta.getColumnCount).each { |i| cols << meta.getColumnName(i) } + rows = [] + while rs.next + row = {} + cols.each { |c| row[c] = rs.getObject(c) } + rows << row + end + rs.close + stmt.close + rows + end + + def self.jdb_exec(conn, sql, params = []) + stmt = conn.prepareStatement(sql) + params.each_with_index { |p, i| stmt.setObject(i + 1, p) } + stmt.executeUpdate + stmt.close + end + def self.log_info(msg) puts "#{Time.now.iso8601} [INFO] #{msg}" append_log("update.log", msg) @@ -44,22 +76,34 @@ module RadioAutomation end def self.open_subs_db - db = SQLite3::Database.new(SUBS_DB) - db.results_as_hash = true + db = jdb_connect(SUBS_DB) + jdb_exec(db, <<-SQL) + CREATE TABLE IF NOT EXISTS shows ( + slug TEXT PRIMARY KEY, + guid TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + feed_url TEXT NOT NULL UNIQUE, + source TEXT DEFAULT 'manual', + opml_import INTEGER DEFAULT 0, + archived INTEGER DEFAULT 1, + created_at TEXT DEFAULT (datetime('now')) + ) + SQL db end def self.open_played_db - db = SQLite3::Database.new(PLAYED_DB) - db.results_as_hash = true - db.execute <<-SQL + db = jdb_connect(PLAYED_DB) + jdb_exec(db, <<-SQL) CREATE TABLE IF NOT EXISTS episodes ( id INTEGER PRIMARY KEY AUTOINCREMENT, show_slug TEXT NOT NULL, guid TEXT NOT NULL, title TEXT, file_path TEXT, - duration_seconds INTEGER, + enclosure_url TEXT, + runlength INTEGER, + played INTEGER DEFAULT 0, played_at TEXT, UNIQUE(show_slug, guid) ) @@ -67,80 +111,85 @@ module RadioAutomation db end - def self.find_audio_files(directory) - return [] unless Dir.exist?(directory) - Dir.glob(File.join(directory, "**", "*")).select do |f| - File.file?(f) && AUDIO_EXTS.any? { |ext| f.end_with?(ext) } - end.sort - end - + # Pick the next unplayed episode for a show, keyed by guid. + # Archived: prefer a local file_path; Live: use enclosure_url. def self.select_unplayed_episode(slug, played_db) - files = find_audio_files(File.join(PODCASTS_DIR, slug)) - return nil if files.empty? - - played_rows = played_db.query_all( - "SELECT file_path, played_at FROM episodes WHERE show_slug = ? AND played_at IS NOT NULL", slug + rows = jdb_query( + played_db, + "SELECT guid, title, file_path, enclosure_url, runlength FROM episodes WHERE show_slug = ? AND played = 0 ORDER BY id ASC LIMIT 1", + [slug] ) - played_map = played_rows.each_with_object({}) { |r, h| h[r["file_path"]] = r["played_at"] } - - unplayed = files.reject { |f| played_map.key?(f) } - return unplayed.first if unplayed.any? - - if played_map.any? - played_map.min_by { |_path, ts| ts.to_s }[0] - else - files.first - end + return nil if rows.empty? + rows.first end - def self.write_pls(filepath, out_path) - abs = File.absolute_path(filepath) - content = "[playlist]\nFile1=#{abs}\nTitle1=Radio Episode\nLength1=-1\nNumberOfEntries=1\nVersion=2\n" + # Build the annotated URI line station.liq consumes. + # Archived -> local file path; Live -> remote enclosure URL. + def self.annotated_uri(ep) + uri = ep["file_path"] || ep["enclosure_url"] + return nil if uri.nil? || uri.to_s.empty? + rl = ep["runlength"].to_i + title = ep["title"].to_s.gsub('"', "'") + "annotate:liq_runlength=\"#{rl}\",liq_title=\"#{title}\":#{uri}" + end + + def self.write_queue_line(slug, line, out_path) FileUtils.mkdir_p(File.dirname(out_path)) - File.write(out_path, content) + File.open(out_path, "w") { |f| f.puts(line) } end - def self.mark_as_played(slug, filepath, played_db) - played_db.execute( - "UPDATE episodes SET played_at = datetime('now') WHERE show_slug = ? AND file_path = ?", - [slug, filepath] + def self.mark_as_played(slug, guid, played_db) + jdb_exec( + played_db, + "UPDATE episodes SET played = 1, played_at = datetime('now') WHERE show_slug = ? AND guid = ?", + [slug, guid] ) end def self.update_all subs_db = open_subs_db played_db = open_played_db - shows = subs_db.query_all("SELECT slug, name FROM shows ORDER BY name") + shows = jdb_query(subs_db, "SELECT slug, name, archived FROM shows ORDER BY name") + queued = 0 + skipped = 0 shows.each do |show| slug = show["slug"] - selected = select_unplayed_episode(slug, played_db) - if selected.nil? - log_info("#{slug}: no audio files found, skipping.") + ep = select_unplayed_episode(slug, played_db) + if ep.nil? + log_info("#{slug}: no unplayed episodes, skipping.") + skipped += 1 next end - out_pls = File.join(PLAYLISTS_DIR, "#{slug}.pls") - write_pls(selected, out_pls) - mark_as_played(slug, selected, played_db) - log_info("#{slug}: queued #{File.basename(selected)}") + line = annotated_uri(ep) + if line.nil? + log_info("#{slug}: episode #{ep['guid']} has no usable URI, skipping.") + skipped += 1 + next + end + out_pls = File.join(PLAYLISTS_DIR, "#{slug}.txt") + write_queue_line(slug, line, out_pls) + mark_as_played(slug, ep["guid"], played_db) + kind = ep["file_path"] ? "downloaded" : "live" + log_info("#{slug}: queued #{ep['title']} [#{kind}] runlength=#{ep['runlength'].to_i}s") + queued += 1 end subs_db.close played_db.close + log_info("=== Update complete: #{queued} queued, #{skipped} skipped ===") end def self.json_summary subs_db = open_subs_db played_db = open_played_db - shows = subs_db.query_all("SELECT slug FROM shows ORDER BY name") + shows = jdb_query(subs_db, "SELECT slug FROM shows ORDER BY name") summary = {} shows.each do |show| slug = show["slug"] - files = find_audio_files(File.join(PODCASTS_DIR, slug)) - played_count = played_db.get_first_value( - "SELECT COUNT(*) FROM episodes WHERE show_slug = ? AND played_at IS NOT NULL", slug - ) - summary[slug] = { "total_files" => files.size, "played_count" => played_count } + total = jdb_query(played_db, "SELECT COUNT(*) AS c FROM episodes WHERE show_slug = ?", [slug]).first["c"].to_i + played = jdb_query(played_db, "SELECT COUNT(*) AS c FROM episodes WHERE show_slug = ? AND played = 1", [slug]).first["c"].to_i + summary[slug] = { "total_episodes" => total, "played_count" => played, "unplayed" => total - played } end subs_db.close played_db.close