CascadiaDigitalCollective/radio
CascadiaDigitalCollective/radio
Watch
1
0
Fork
You've already forked radio
 0
mirror of https://github.com/mistergibson/radio.git synced 2026-09-08 22:09:51 -07:00
Code Issues Projects Releases Packages Wiki Activity

Overhaul

Browse source
This commit is contained in:
G. Gibson 2026-09-01 15:27:17 -07:00
commit 04098bc151
6 changed files with 385 additions and 187 deletions
Download patch file Download diff file Expand all files Collapse all files

81
fetch_podcasts.py
View file

@ -8,6 +8,9 @@ defined in config.json ("storage" key), keeping the boot drive clean.
Filters out video podcasts: only shows whose latest enclosure has an
audio/* MIME type are registered.
Archived shows (archived=1) have episodes downloaded to disk; non-archived
shows (archived=0) store only the enclosure URL for live streaming.
"""
import argparse
@ -17,6 +20,7 @@ import re
import shutil
import sqlite3
import sys
import uuid
import xml.etree.ElementTree as ET
from pathlib import Path
from urllib.parse import quote
@ -28,6 +32,7 @@ ROOT = Path(__file__).resolve().parent
CONFIG_PATH = ROOT / "config.json"
AUDIO_EXTS = {".mp3", ".m4a"}
VIDEO_EXTS = {".mp4", ".mov", ".avi", ".webm", ".mkv"}
STATE_DIR = None
SUBS_DB = None
@ -76,10 +81,12 @@ def open_subs_db():
conn.execute("""
CREATE TABLE IF NOT EXISTS shows (
slug TEXT PRIMARY KEY,
guid TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
feed_url TEXT NOT NULL UNIQUE,
source TEXT DEFAULT 'manual',
opml_import INTEGER DEFAULT 0,
archived INTEGER DEFAULT 1,
created_at TEXT DEFAULT (datetime('now'))
)
""")
@ -96,8 +103,10 @@ def open_played_db():
guid TEXT NOT NULL,
title TEXT,
file_path TEXT,
duration_seconds INTEGER,
played_at TEXT DEFAULT (datetime('now')),
enclosure_url TEXT,
runlength INTEGER,
played INTEGER DEFAULT 0,
played_at TEXT,
UNIQUE(show_slug, guid)
)
""")
@ -108,6 +117,9 @@ def slugify(name):
s = re.sub(r"[^a-z0-9]+", "_", name.lower()).strip("_")
return s[:60] or "show"
def gen_uuid():
return str(uuid.uuid4())
def is_audio_feed(parsed):
"""Check whether the feed's latest episode has an audio enclosure.
Returns True if audio, False if video or unknown.
@ -123,13 +135,12 @@ def is_audio_feed(parsed):
return True
if mime_type.startswith("video/"):
return False
# Unknown type: check URL extension as fallback
url = (enclosures[0].get("href") or "").lower()
if any(url.endswith(ext) for ext in AUDIO_EXTS):
return True
if any(url.endswith(ext) for ext in (".mp4", ".mov", ".avi", ".webm", ".mkv")):
if any(url.endswith(ext) for ext in VIDEO_EXTS):
return False
return True # Default to allowing if undetermined
return True
def gpodder_sync(cfg):
g = cfg["gpodder"]
@ -174,9 +185,14 @@ def parse_opml(xml_string):
for outline in root.iter("outline"):
feed_url = (outline.attrib.get("xmlUrl") or "").strip()
name = (outline.attrib.get("text") or "").strip()
guid = (outline.attrib.get("guid") or "").strip()
if not feed_url or not re.match(r"^https?://", feed_url):
continue
shows.append({"name": name, "feed_url": feed_url})
shows.append({
"name": name,
"feed_url": feed_url,
"guid": guid if guid else None,
})
return shows
def register_remote_shows(remote_shows):
@ -188,7 +204,6 @@ def register_remote_shows(remote_shows):
existing = db.execute("SELECT slug FROM shows WHERE slug = ?", (slug,)).fetchone()
if existing is not None:
continue
# Audio-only filter: check the feed before registering
parsed = fetch_feed(show["feed_url"])
if parsed is None:
log.warning("Skipping '%s': could not fetch feed.", show["name"])
@ -197,9 +212,10 @@ def register_remote_shows(remote_shows):
log.info("Skipping '%s' (%s): video podcast, not audio.", show["name"], slug)
skipped_video += 1
continue
guid = show["guid"] or gen_uuid()
db.execute(
"INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)",
(slug, show["name"], show["feed_url"]),
"INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'gpodder', 0, 1)",
(slug, guid, show["name"], show["feed_url"]),
)
log.info("Registered new show: %s (%s)", show["name"], slug)
added += 1
@ -281,6 +297,12 @@ def safe_filename(title, fallback):
name = re.sub(r"[^\w\s.-]", "", title or "").strip().replace(" ", "_")
return (name[:120] or fallback) + ".mp3"
def show_archived(subs_db, slug):
row = subs_db.execute("SELECT archived FROM shows WHERE slug = ?", (slug,)).fetchone()
if row is None:
return True
return row["archived"] == 1
def fetch_show_episodes(slug, name, feed_url):
dest_dir = PODCASTS_DIR / slug
dest_dir.mkdir(parents=True, exist_ok=True)
@ -293,6 +315,7 @@ def fetch_show_episodes(slug, name, feed_url):
for row in played_db.execute("SELECT guid FROM episodes WHERE show_slug = ?", (slug,))
}
subs_db = open_subs_db()
archived = show_archived(subs_db, slug)
new_count = 0
for entry in parsed.entries:
guid = entry.get("id") or entry.get("link") or entry.get("title", "")
@ -301,7 +324,6 @@ def fetch_show_episodes(slug, name, feed_url):
enclosures = entry.get("enclosures") or []
if not enclosures:
continue
# Per-episode audio check (catches mixed-content feeds)
mime = (enclosures[0].get("type") or "").lower()
if mime.startswith("video/"):
continue
@ -309,18 +331,27 @@ def fetch_show_episodes(slug, name, feed_url):
if not audio_url:
continue
title = entry.get("title", "untitled")
duration = extract_duration(entry)
if archived:
filename = safe_filename(title, guid[-20:])
file_path = download_episode(audio_url, dest_dir, filename)
if file_path is None:
continue
duration = extract_duration(entry)
played_db.execute(
"INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, duration_seconds, played_at) "
"VALUES (?, ?, ?, ?, ?, NULL)",
(slug, guid, title, file_path, duration),
"INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, enclosure_url, runlength, played) "
"VALUES (?, ?, ?, ?, ?, ?, 0)",
(slug, guid, title, file_path, audio_url, duration),
)
else:
played_db.execute(
"INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, enclosure_url, runlength, played) "
"VALUES (?, ?, ?, NULL, ?, ?, 0)",
(slug, guid, title, audio_url, duration),
)
new_count += 1
log.info(" New episode: %s [%s]", title, filename)
kind = "downloaded" if archived else "live"
log.info(" New episode: %s [%s]", title, kind)
played_db.commit()
played_db.close()
subs_db.close()
@ -343,16 +374,16 @@ def fetch_all_episodes():
def list_shows(detail=False):
db = open_subs_db()
rows = db.execute(
"SELECT slug, name, feed_url, source, opml_import FROM shows ORDER BY name"
"SELECT slug, name, feed_url, source, opml_import, archived FROM shows ORDER BY name"
).fetchall()
db.close()
if not rows:
print("No shows registered.")
return
print(f"{'SLUG':<30} {'PROTECTED':<10} NAME")
print(f"{'SLUG':<30} {'ARCHIVED':<10} {'SOURCE':<10} NAME")
for r in rows:
prot = "yes" if r["opml_import"] else "no"
line = f"{r['slug']:<30} {prot:<10} {r['name']}"
arch = "yes" if r["archived"] == 1 else "no"
line = f"{r['slug']:<30} {arch:<10} {r['source']:<10} {r['name']}"
if detail:
line += f"\n{'':<50} {r['feed_url']}"
print(line)
@ -367,10 +398,11 @@ def add_show(feed_url):
return
name = parsed.feed["title"]
slug = slugify(name)
guid = gen_uuid()
db = open_subs_db()
db.execute(
"INSERT OR IGNORE INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'manual', 0)",
(slug, name, feed_url),
"INSERT OR IGNORE INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'manual', 0, 1)",
(slug, guid, name, feed_url),
)
db.commit()
db.close()
@ -381,7 +413,7 @@ def remove_show_data(slug):
pod_dir = PODCASTS_DIR / slug
if pod_dir.exists():
shutil.rmtree(pod_dir)
pls = PLAYLISTS_DIR / f"{slug}.pls"
pls = PLAYLISTS_DIR / f"{slug}.txt"
if pls.exists():
pls.unlink()
@ -425,9 +457,10 @@ def import_opml(path):
log.info("OPML import: skipping '%s' (%s): video podcast.", show["name"], slug)
skipped_video += 1
continue
guid = show["guid"] or gen_uuid()
db.execute(
"INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)",
(slug, show["name"], show["feed_url"]),
"INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'opml', 1, 1)",
(slug, guid, show["name"], show["feed_url"]),
)
added += 1
db.commit()

157
fetch_podcasts.rb
View file

@ -5,12 +5,15 @@ require "net/http"
require "uri"
require "cgi"
require "json"
require "sqlite3"
require "jdbc/sqlite3"
require "rexml/document"
require "digest/md5"
require "securerandom"
require "fileutils"
require "optparse"
Jdbc::SQLite3.load_driver
module RadioAutomation
ROOT = File.expand_path("..", __dir__)
CONFIG_PATH = File.join(ROOT, "config.json")
@ -37,6 +40,37 @@ module RadioAutomation
self.PLAYLISTS_DIR = File.join(@storage, "playlists")
end
# --- JDBC connection helpers ---------------------------------------------
def self.jdb_connect(db_file)
java.sql.DriverManager.getConnection("jdbc:sqlite:#{db_file}")
end
def self.jdb_query(conn, sql, params = [])
stmt = conn.prepareStatement(sql)
params.each_with_index { |p, i| stmt.setObject(i + 1, p) }
rs = stmt.executeQuery
cols = []
meta = rs.getMetaData
(1..meta.getColumnCount).each { |i| cols << meta.getColumnName(i) }
rows = []
while rs.next
row = {}
cols.each { |c| row[c] = rs.getObject(c) }
rows << row
end
rs.close
stmt.close
rows
end
def self.jdb_exec(conn, sql, params = [])
stmt = conn.prepareStatement(sql)
params.each_with_index { |p, i| stmt.setObject(i + 1, p) }
stmt.executeUpdate
stmt.close
end
def self.log_info(msg)
puts "#{Time.now.iso8601} [INFO] #{msg}"
append_log("fetch.log", msg)
@ -68,16 +102,21 @@ module RadioAutomation
s[0, 60] || "show"
end
def self.gen_uuid
SecureRandom.uuid
end
def self.open_subs_db
db = SQLite3::Database.new(SUBS_DB)
db.results_as_hash = true
db.execute <<-SQL
db = jdb_connect(SUBS_DB)
jdb_exec(db, <<-SQL)
CREATE TABLE IF NOT EXISTS shows (
slug TEXT PRIMARY KEY,
guid TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
feed_url TEXT NOT NULL UNIQUE,
source TEXT DEFAULT 'manual',
opml_import INTEGER DEFAULT 0,
archived INTEGER DEFAULT 1,
created_at TEXT DEFAULT (datetime('now'))
)
SQL
@ -85,16 +124,17 @@ module RadioAutomation
end
def self.open_played_db
db = SQLite3::Database.new(PLAYED_DB)
db.results_as_hash = true
db.execute <<-SQL
db = jdb_connect(PLAYED_DB)
jdb_exec(db, <<-SQL)
CREATE TABLE IF NOT EXISTS episodes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
show_slug TEXT NOT NULL,
guid TEXT NOT NULL,
title TEXT,
file_path TEXT,
duration_seconds INTEGER,
enclosure_url TEXT,
runlength INTEGER,
played INTEGER DEFAULT 0,
played_at TEXT,
UNIQUE(show_slug, guid)
)
@ -109,35 +149,30 @@ module RadioAutomation
entry_xml.match(/enclosure[^>]*url="([^"]*)"[^>]*type="([^"]*)"/i)
if enc_m
if enc_m.pre_match.include?("type=") && enc_m.post_match.empty?
# First pattern matched: group 1 = type, group 2 = url
return [enc_m[1], enc_m[2]]
else
# Second pattern matched: group 1 = url, group 2 = type
return [enc_m[2], enc_m[1]]
end
end
# Fallback: just grab url without type
url_only = entry_xml.match(/enclosure[^>]*url="([^"]*)"/i)
[nil, url_only[1]] if url_only
end
def self.is_audio_entry?(entry_xml)
mime, url = enclosure_mime_and_url(entry_xml)
return true if mime.nil? && url.nil? # No enclosure; assume ok
return true if mime.nil? && url.nil?
mime_l = mime.to_s.downcase
return true if mime_l.start_with?("audio/")
return false if mime_l.start_with?("video/")
# Fall back to URL extension
url_l = url.to_s.downcase
return true if AUDIO_EXTS.any? { |ext| url_l.end_with?(ext) }
return false if VIDEO_EXTS.any? { |ext| url_l.end_with?(ext) }
true # Undetermined: allow
true
end
def self.is_audio_feed?(raw_xml)
# Grab the first <item> or <entry> block
m = raw_xml.match(/<(?:item|entry)[^>]*>(.*?)<\/(?:item|entry)>/mi)
return true unless m # No items; let through
return true unless m
is_audio_entry?(m[1])
end
@ -195,8 +230,9 @@ module RadioAutomation
REXML::XPath.each(doc, "//outline[@xmlUrl]") do |node|
feed_url = node.attributes["xmlUrl"].to_s.strip
name = node.attributes["text"].to_s.strip
guid = node.attributes["guid"].to_s.strip
next unless feed_url =~ /\Ahttps?:\/\//
shows << { "name" => name, "feed_url" => feed_url }
shows << { "name" => name, "feed_url" => feed_url, "guid" => (guid.empty? ? nil : guid) }
end
shows
rescue REXML::ParseException => e
@ -210,8 +246,8 @@ module RadioAutomation
skipped_video = 0
remote_shows.each do |show|
slug = slugify(show["name"])
existing = db.get_first_value("SELECT slug FROM shows WHERE slug = ?", slug)
next unless existing.nil?
existing = jdb_query(db, "SELECT slug FROM shows WHERE slug = ?", [slug])
next unless existing.empty?
raw = fetch_feed(show["feed_url"])
if raw.nil?
@ -223,9 +259,11 @@ module RadioAutomation
skipped_video += 1
next
end
db.execute(
"INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'gpodder', 0)",
[slug, show["name"], show["feed_url"]]
guid = show["guid"] || gen_uuid
jdb_exec(
db,
"INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'gpodder', 0, 1)",
[slug, guid, show["name"], show["feed_url"]]
)
log_info("Registered new show: #{show['name']} (#{slug})")
added += 1
@ -238,12 +276,12 @@ module RadioAutomation
def self.prune_stale_shows(remote_shows)
db = open_subs_db
remote_slugs = remote_shows.map { |s| slugify(s["name"]) }
stale = db.query_all("SELECT slug, name FROM shows WHERE source = 'gpodder' AND opml_import = 0")
stale = jdb_query(db, "SELECT slug, name FROM shows WHERE source = 'gpodder' AND opml_import = 0")
removed = 0
stale.each do |row|
next if remote_slugs.include?(row["slug"])
remove_show_data(row["slug"])
db.execute("DELETE FROM shows WHERE slug = ?", [row["slug"]])
jdb_exec(db, "DELETE FROM shows WHERE slug = ?", [row["slug"]])
log_info("Pruned stale show: #{row['name']} (#{row['slug']})")
removed += 1
end
@ -318,14 +356,21 @@ module RadioAutomation
"#{name || fallback}.mp3"
end
def self.show_archived?(subs_db, slug)
rows = jdb_query(subs_db, "SELECT archived FROM shows WHERE slug = ?", [slug])
rows.empty? ? true : (rows.first["archived"] == 1)
end
def self.fetch_show_episodes(slug, name, feed_url)
dest_dir = File.join(PODCASTS_DIR, slug)
raw = fetch_feed(feed_url)
return 0 if raw.nil?
played_db = open_played_db
seen = played_db.query("SELECT guid FROM episodes WHERE show_slug = ?", slug).map { |r| r["guid"] }
seen_rows = jdb_query(played_db, "SELECT guid FROM episodes WHERE show_slug = ?", [slug])
seen = seen_rows.map { |r| r["guid"] }
subs_db = open_subs_db
archived = show_archived?(subs_db, slug)
new_count = 0
raw.scan(/<(?:item|entry)[^>]*>(.*?)<\/(?:item|entry)>/mi).each do |(entry_xml)|
@ -333,7 +378,6 @@ module RadioAutomation
guid = guid_m ? (guid_m[1] || guid_m[2] || guid_m[3]).strip : Digest::MD5.hexdigest(entry_xml[0, 200])
next if seen.include?(guid)
# Per-episode audio check
unless is_audio_entry?(entry_xml)
next
end
@ -343,17 +387,26 @@ module RadioAutomation
title_m = entry_xml.match(/<title[^>]*>([^<]*)<\/title>/i)
title = title_m ? title_m[1].strip : "untitled"
duration = extract_duration(entry_xml)
if archived
filename = safe_filename(title, guid[-20..])
file_path = download_episode(audio_url, dest_dir, filename)
next if file_path.nil?
duration = extract_duration(entry_xml)
played_db.execute(
"INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, duration_seconds, played_at) VALUES (?, ?, ?, ?, ?, NULL)",
[slug, guid, title, file_path, duration]
jdb_exec(
played_db,
"INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, enclosure_url, runlength, played) VALUES (?, ?, ?, ?, ?, ?, 0)",
[slug, guid, title, file_path, audio_url, duration]
)
else
jdb_exec(
played_db,
"INSERT OR IGNORE INTO episodes (show_slug, guid, title, file_path, enclosure_url, runlength, played) VALUES (?, ?, ?, NULL, ?, ?, 0)",
[slug, guid, title, audio_url, duration]
)
end
new_count += 1
log_info(" New episode: #{title} [#{filename}]")
log_info(" New episode: #{title} [#{archived ? 'downloaded' : 'live'}]")
end
played_db.close
@ -363,7 +416,7 @@ module RadioAutomation
def self.fetch_all_episodes
db = open_subs_db
shows = db.query_all("SELECT slug, name, feed_url FROM shows ORDER BY name")
shows = jdb_query(db, "SELECT slug, name, feed_url FROM shows ORDER BY name")
db.close
total_new = 0
shows.each do |show|
@ -379,16 +432,16 @@ module RadioAutomation
def self.list_shows(detail: false)
db = open_subs_db
rows = db.query_all("SELECT slug, name, feed_url, source, opml_import FROM shows ORDER BY name")
rows = jdb_query(db, "SELECT slug, name, feed_url, source, opml_import, archived FROM shows ORDER BY name")
db.close
if rows.empty?
puts "No shows registered."
return
end
puts format("%-30s %-10s %s", "SLUG", "PROTECTED", "NAME")
puts format("%-30s %-10s %-8s %s", "SLUG", "ARCHIVED", "SOURCE", "NAME")
rows.each do |r|
prot = r["opml_import"] ? "yes" : "no"
line = format("%-30s %-10s %s", r["slug"], prot, r["name"])
arch = r["archived"] == 1 ? "yes" : "no"
line = format("%-30s %-10s %-8s %s", r["slug"], arch, r["source"], r["name"])
line += "\n" + (" " * 50) + r["feed_url"] if detail
puts line
end
@ -412,10 +465,12 @@ module RadioAutomation
end
name = title_m[1].strip
slug = slugify(name)
guid = gen_uuid
db = open_subs_db
db.execute(
"INSERT OR IGNORE INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'manual', 0)",
[slug, name, feed_url]
jdb_exec(
db,
"INSERT OR IGNORE INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'manual', 0, 1)",
[slug, guid, name, feed_url]
)
db.close
log_info("Added show: #{name} (#{slug})")
@ -431,18 +486,18 @@ module RadioAutomation
def self.delete_show(slug)
db = open_subs_db
row = db.get_first_hash("SELECT name FROM shows WHERE slug = ?", slug)
if row.nil?
rows = jdb_query(db, "SELECT name FROM shows WHERE slug = ?", [slug])
if rows.empty?
log_error("No show found with slug '#{slug}'.")
return
end
remove_show_data(slug)
db.execute("DELETE FROM shows WHERE slug = ?", [slug])
jdb_exec(db, "DELETE FROM shows WHERE slug = ?", [slug])
db.close
played_db = open_played_db
played_db.execute("DELETE FROM episodes WHERE show_slug = ?", [slug])
jdb_exec(played_db, "DELETE FROM episodes WHERE show_slug = ?", [slug])
played_db.close
log_info("Deleted show: #{row['name']} (#{slug})")
log_info("Deleted show: #{rows.first['name']} (#{slug})")
end
def self.import_opml(path)
@ -453,8 +508,8 @@ module RadioAutomation
skipped_video = 0
shows.each do |show|
slug = slugify(show["name"])
existing = db.get_first_value("SELECT slug FROM shows WHERE slug = ?", slug)
next unless existing.nil?
existing = jdb_query(db, "SELECT slug FROM shows WHERE slug = ?", [slug])
next unless existing.empty?
raw = fetch_feed(show["feed_url"])
if raw.nil?
@ -466,9 +521,11 @@ module RadioAutomation
skipped_video += 1
next
end
db.execute(
"INSERT INTO shows (slug, name, feed_url, source, opml_import) VALUES (?, ?, ?, 'opml', 1)",
[slug, show["name"], show["feed_url"]]
guid = show["guid"] || gen_uuid
jdb_exec(
db,
"INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'opml', 1, 1)",
[slug, guid, show["name"], show["feed_url"]]
)
added += 1
end

27
install_for_jruby
View file

@ -21,12 +21,27 @@ echo "Install dir: ${SCRIPT_DIR}"
echo "Using jruby: ${JRuby_BIN}"
echo ""
# --- Verify required source files exist ------------------------------------
for req in station.liq fetch_podcasts.rb update_playlists.rb; do
if [ ! -f "${SCRIPT_DIR}/${req}" ]; then
echo "ERROR: required file '${req}' not found in ${SCRIPT_DIR}" >&2
exit 1
fi
done
# --- Prompt for storage path ---------------------------------------------
DEFAULT_STORAGE="/srv/radio-storage"
read -rp "Enter storage path for media/state/logs [${DEFAULT_STORAGE}]: " STORAGE_PATH
STORAGE_PATH="${STORAGE_PATH:-$DEFAULT_STORAGE}"
mkdir -p "${STORAGE_PATH}"/{music,podcasts,jingles,announcements,state,playlists,logs}
chown -R liquidsoap:liquidsoap "${STORAGE_PATH}" 2>/dev/null || true
# Media dirs are read by the liquidsoap user at stream time
chown -R liquidsoap:liquidsoap "${STORAGE_PATH}"/{music,podcasts,jingles,announcements} 2>/dev/null || true
# state/ and playlists/ are written by the cron jobs (run as root) and read by liquidsoap
chown root:liquidsoap "${STORAGE_PATH}"/{state,playlists} 2>/dev/null || true
chmod 775 "${STORAGE_PATH}"/{state,playlists} 2>/dev/null || true
# logs/ is appended to by cron
chown root:liquidsoap "${STORAGE_PATH}/logs" 2>/dev/null || true
chmod 775 "${STORAGE_PATH}/logs" 2>/dev/null || true
# --- Collect Icecast credentials ------------------------------------------
read -rp "Icecast host [127.0.0.1]: " IC_HOST
@ -78,10 +93,12 @@ chmod 600 "$CONFIG_FILE"
echo "Wrote ${CONFIG_FILE}"
# --- Gem check (install one at a time to avoid memory limits) --------------
for gem in sqlite3 json; do
# Use jdbc-sqlite3, not sqlite3: the latter needs a native C extension that
# fails to build on JRuby. jdbc-sqlite3 ships the SQLite JDBC driver as a JAR.
for gem in jdbc-sqlite3 json; do
if ! "${JRuby_BIN}" -e "require '${gem}'" >/dev/null 2>&1; then
echo "Installing gem: ${gem}"
gem install "${gem}"
"${JRuby_BIN}" -S gem install "${gem}"
fi
done
@ -109,7 +126,9 @@ echo "Enabled systemd unit: ${SERVICE_NAME}"
CRON_FETCH="0 * * * * cd ${SCRIPT_DIR} && ${JRuby_BIN} fetch_podcasts.rb >> ${STORAGE_PATH}/logs/cron.log 2>&1"
CRON_PLAYLISTS="30 * * * * cd ${SCRIPT_DIR} && ${JRuby_BIN} update_playlists.rb >> ${STORAGE_PATH}/logs/cron.log 2>&1"
(crontab -l 2>/dev/null | grep -v "fetch_podcasts.rb\|update_playlists.rb"; echo "$CRON_FETCH"; echo "$CRON_PLAYLISTS") | crontab -
# Guard against pipefail killing us when crontab is empty or has no matching lines
EXISTING_CRON=$(crontab -l 2>/dev/null | grep -v "fetch_podcasts.rb\|update_playlists.rb" || true)
( [ -n "$EXISTING_CRON" ] && echo "$EXISTING_CRON"; echo "$CRON_FETCH"; echo "$CRON_PLAYLISTS" ) | crontab -
echo "Installed cron jobs:"
echo " $CRON_FETCH"
echo " $CRON_PLAYLISTS"

21
install_for_python
View file

@ -20,12 +20,27 @@ echo "=== ${SERVICE_NAME} installer (Python) ==="
echo "Install dir: ${SCRIPT_DIR}"
echo ""
# --- Verify required source files exist ------------------------------------
for req in station.liq fetch_podcasts.py update_playlists.py; do
if [ ! -f "${SCRIPT_DIR}/${req}" ]; then
echo "ERROR: required file '${req}' not found in ${SCRIPT_DIR}" >&2
exit 1
fi
done
# --- Prompt for storage path ---------------------------------------------
DEFAULT_STORAGE="/srv/radio-storage"
read -rp "Enter storage path for media/state/logs [${DEFAULT_STORAGE}]: " STORAGE_PATH
STORAGE_PATH="${STORAGE_PATH:-$DEFAULT_STORAGE}"
mkdir -p "${STORAGE_PATH}"/{music,podcasts,jingles,announcements,state,playlists,logs}
chown -R liquidsoap:liquidsoap "${STORAGE_PATH}" 2>/dev/null || true
# Media dirs are read by the liquidsoap user at stream time
chown -R liquidsoap:liquidsoap "${STORAGE_PATH}"/{music,podcasts,jingles,announcements} 2>/dev/null || true
# state/ and playlists/ are written by the cron jobs (run as root) and read by liquidsoap
chown root:liquidsoap "${STORAGE_PATH}"/{state,playlists} 2>/dev/null || true
chmod 775 "${STORAGE_PATH}"/{state,playlists} 2>/dev/null || true
# logs/ is appended to by cron
chown root:liquidsoap "${STORAGE_PATH}/logs" 2>/dev/null || true
chmod 775 "${STORAGE_PATH}/logs" 2>/dev/null || true
# --- Collect Icecast credentials ------------------------------------------
read -rp "Icecast host [127.0.0.1]: " IC_HOST
@ -111,7 +126,9 @@ echo "Enabled systemd unit: ${SERVICE_NAME}"
CRON_FETCH="0 * * * * cd ${SCRIPT_DIR} && ${PYBIN} fetch_podcasts.py >> ${STORAGE_PATH}/logs/cron.log 2>&1"
CRON_PLAYLISTS="30 * * * * cd ${SCRIPT_DIR} && ${PYBIN} update_playlists.py >> ${STORAGE_PATH}/logs/cron.log 2>&1"
(crontab -l 2>/dev/null | grep -v "fetch_podcasts.py\|update_playlists.py"; echo "$CRON_FETCH"; echo "$CRON_PLAYLISTS") | crontab -
# Guard against pipefail killing us when crontab is empty or has no matching lines
EXISTING_CRON=$(crontab -l 2>/dev/null | grep -v "fetch_podcasts.py\|update_playlists.py" || true)
( [ -n "$EXISTING_CRON" ] && echo "$EXISTING_CRON"; echo "$CRON_FETCH"; echo "$CRON_PLAYLISTS" ) | crontab -
echo "Installed cron jobs:"
echo " $CRON_FETCH"
echo " $CRON_PLAYLISTS"

117
update_playlists.py
View file

@ -1,7 +1,12 @@
#!/usr/bin/env python3
"""
update_playlists.py - Regenerate per-show .pls playlists based on playback history.
Runs via cron hourly at :30. All data under the storage path from config.json.
update_playlists.py - Select the next unplayed episode per show and write an
annotated URI line for station.liq to consume. Runs via cron hourly at :30.
Selection is driven by the episodes table (keyed on guid + played flag), not
by scanning the filesystem, so it works identically for archived shows
(local file_path) and live shows (remote enclosure_url). All data under the
storage path from config.json.
"""
import argparse
@ -14,8 +19,6 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parent
CONFIG_PATH = ROOT / "config.json"
AUDIO_EXTS = {".mp3", ".m4a"}
STATE_DIR = None
SUBS_DB = None
PLAYED_DB = None
@ -55,6 +58,19 @@ def _setup_logging():
def open_subs_db():
conn = sqlite3.connect(SUBS_DB)
conn.row_factory = sqlite3.Row
conn.execute("""
CREATE TABLE IF NOT EXISTS shows (
slug TEXT PRIMARY KEY,
guid TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
feed_url TEXT NOT NULL UNIQUE,
source TEXT DEFAULT 'manual',
opml_import INTEGER DEFAULT 0,
archived INTEGER DEFAULT 1,
created_at TEXT DEFAULT (datetime('now'))
)
""")
conn.commit()
return conn
def open_played_db():
@ -67,7 +83,9 @@ def open_played_db():
guid TEXT NOT NULL,
title TEXT,
file_path TEXT,
duration_seconds INTEGER,
enclosure_url TEXT,
runlength INTEGER,
played INTEGER DEFAULT 0,
played_at TEXT,
UNIQUE(show_slug, guid)
)
@ -75,60 +93,64 @@ def open_played_db():
conn.commit()
return conn
def find_audio_files(directory):
results = []
if not directory.exists():
return results
for p in sorted(directory.rglob("*")):
if p.is_file() and p.suffix.lower() in AUDIO_EXTS:
results.append(str(p))
return results
def select_unplayed_episode(slug, played_db):
files = find_audio_files(PODCASTS_DIR / slug)
if not files:
return None
played_rows = played_db.execute(
"SELECT file_path, played_at FROM episodes WHERE show_slug = ? AND played_at IS NOT NULL",
"""Pick the next unplayed episode for a show, keyed by guid.
Archived -> local file_path; Live -> remote enclosure_url."""
row = played_db.execute(
"SELECT guid, title, file_path, enclosure_url, runlength "
"FROM episodes WHERE show_slug = ? AND played = 0 ORDER BY id ASC LIMIT 1",
(slug,),
).fetchall()
played_paths = {row["file_path"]: row["played_at"] for row in played_rows}
unplayed = [f for f in files if f not in played_paths]
if unplayed:
return unplayed[0]
if played_paths:
return min(played_paths.items(), key=lambda kv: (kv[1] or ""))[0]
return files[0]
).fetchone()
return row
def write_pls(filepath, out_path):
abs_path = str(Path(filepath).resolve())
content = f"[playlist]\nFile1={abs_path}\nTitle1=Radio Episode\nLength1=-1\nNumberOfEntries=1\nVersion=2\n"
def annotated_uri(ep):
"""Build the annotate: URI line station.liq consumes.
Archived -> local file path; Live -> remote enclosure URL."""
uri = ep["file_path"] or ep["enclosure_url"]
if not uri:
return None
rl = int(ep["runlength"] or 0)
title = (ep["title"] or "").replace('"', "'")
return f'annotate:liq_runlength="{rl}",liq_title="{title}":{uri}'
def write_queue_line(slug, line, out_path):
out_path.parent.mkdir(parents=True, exist_ok=True)
out_path.write_text(content)
out_path.write_text(line + "\n")
def mark_as_played(slug, filepath, played_db):
def mark_as_played(slug, guid, played_db):
played_db.execute(
"UPDATE episodes SET played_at = datetime('now') WHERE show_slug = ? AND file_path = ?",
(slug, filepath),
"UPDATE episodes SET played = 1, played_at = datetime('now') WHERE show_slug = ? AND guid = ?",
(slug, guid),
)
played_db.commit()
def update_all():
subs_db = open_subs_db()
played_db = open_played_db()
shows = subs_db.execute("SELECT slug, name FROM shows ORDER BY name").fetchall()
shows = subs_db.execute("SELECT slug, name, archived FROM shows ORDER BY name").fetchall()
queued = 0
skipped = 0
for show in shows:
slug = show["slug"]
selected = select_unplayed_episode(slug, played_db)
if selected is None:
log.info("%s: no audio files found, skipping.", slug)
ep = select_unplayed_episode(slug, played_db)
if ep is None:
log.info("%s: no unplayed episodes, skipping.", slug)
skipped += 1
continue
out_pls = PLAYLISTS_DIR / f"{slug}.pls"
write_pls(selected, out_pls)
mark_as_played(slug, selected, played_db)
log.info("%s: queued %s", slug, Path(selected).name)
line = annotated_uri(ep)
if line is None:
log.info("%s: episode %s has no usable URI, skipping.", slug, ep["guid"])
skipped += 1
continue
out_txt = PLAYLISTS_DIR / f"{slug}.txt"
write_queue_line(slug, line, out_txt)
mark_as_played(slug, ep["guid"], played_db)
kind = "downloaded" if ep["file_path"] else "live"
log.info("%s: queued %s [%s] runlength=%ss", slug, ep["title"], kind, int(ep["runlength"] or 0))
queued += 1
subs_db.close()
played_db.close()
log.info("=== Update complete: %d queued, %d skipped ===", queued, skipped)
def json_summary():
subs_db = open_subs_db()
@ -137,12 +159,13 @@ def json_summary():
summary = {}
for show in shows:
slug = show["slug"]
files = find_audio_files(PODCASTS_DIR / slug)
played_count = played_db.execute(
"SELECT COUNT(*) as c FROM episodes WHERE show_slug = ? AND played_at IS NOT NULL",
(slug,),
total = played_db.execute(
"SELECT COUNT(*) as c FROM episodes WHERE show_slug = ?", (slug,)
).fetchone()["c"]
summary[slug] = {"total_files": len(files), "played_count": played_count}
played = played_db.execute(
"SELECT COUNT(*) as c FROM episodes WHERE show_slug = ? AND played = 1", (slug,)
).fetchone()["c"]
summary[slug] = {"total_episodes": total, "played_count": played, "unplayed": total - played}
subs_db.close()
played_db.close()
print(json.dumps(summary, indent=2))

153
update_playlists.rb
View file

@ -2,14 +2,15 @@
# frozen_string_literal: true
require "json"
require "sqlite3"
require "jdbc/sqlite3"
require "fileutils"
require "optparse"
Jdbc::SQLite3.load_driver
module RadioAutomation
ROOT = File.expand_path("..", __dir__)
CONFIG_PATH = File.join(ROOT, "config.json")
AUDIO_EXTS = [".mp3", ".m4a"]
STORAGE_DIR = nil
STATE_DIR = nil
@ -31,6 +32,37 @@ module RadioAutomation
self.LOGS_DIR = File.join(@storage, "logs")
end
# --- JDBC connection helpers ---------------------------------------------
def self.jdb_connect(db_file)
java.sql.DriverManager.getConnection("jdbc:sqlite:#{db_file}")
end
def self.jdb_query(conn, sql, params = [])
stmt = conn.prepareStatement(sql)
params.each_with_index { |p, i| stmt.setObject(i + 1, p) }
rs = stmt.executeQuery
cols = []
meta = rs.getMetaData
(1..meta.getColumnCount).each { |i| cols << meta.getColumnName(i) }
rows = []
while rs.next
row = {}
cols.each { |c| row[c] = rs.getObject(c) }
rows << row
end
rs.close
stmt.close
rows
end
def self.jdb_exec(conn, sql, params = [])
stmt = conn.prepareStatement(sql)
params.each_with_index { |p, i| stmt.setObject(i + 1, p) }
stmt.executeUpdate
stmt.close
end
def self.log_info(msg)
puts "#{Time.now.iso8601} [INFO] #{msg}"
append_log("update.log", msg)
@ -44,22 +76,34 @@ module RadioAutomation
end
def self.open_subs_db
db = SQLite3::Database.new(SUBS_DB)
db.results_as_hash = true
db = jdb_connect(SUBS_DB)
jdb_exec(db, <<-SQL)
CREATE TABLE IF NOT EXISTS shows (
slug TEXT PRIMARY KEY,
guid TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
feed_url TEXT NOT NULL UNIQUE,
source TEXT DEFAULT 'manual',
opml_import INTEGER DEFAULT 0,
archived INTEGER DEFAULT 1,
created_at TEXT DEFAULT (datetime('now'))
)
SQL
db
end
def self.open_played_db
db = SQLite3::Database.new(PLAYED_DB)
db.results_as_hash = true
db.execute <<-SQL
db = jdb_connect(PLAYED_DB)
jdb_exec(db, <<-SQL)
CREATE TABLE IF NOT EXISTS episodes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
show_slug TEXT NOT NULL,
guid TEXT NOT NULL,
title TEXT,
file_path TEXT,
duration_seconds INTEGER,
enclosure_url TEXT,
runlength INTEGER,
played INTEGER DEFAULT 0,
played_at TEXT,
UNIQUE(show_slug, guid)
)
@ -67,80 +111,85 @@ module RadioAutomation
db
end
def self.find_audio_files(directory)
return [] unless Dir.exist?(directory)
Dir.glob(File.join(directory, "**", "*")).select do |f|
File.file?(f) && AUDIO_EXTS.any? { |ext| f.end_with?(ext) }
end.sort
end
# Pick the next unplayed episode for a show, keyed by guid.
# Archived: prefer a local file_path; Live: use enclosure_url.
def self.select_unplayed_episode(slug, played_db)
files = find_audio_files(File.join(PODCASTS_DIR, slug))
return nil if files.empty?
played_rows = played_db.query_all(
"SELECT file_path, played_at FROM episodes WHERE show_slug = ? AND played_at IS NOT NULL", slug
rows = jdb_query(
played_db,
"SELECT guid, title, file_path, enclosure_url, runlength FROM episodes WHERE show_slug = ? AND played = 0 ORDER BY id ASC LIMIT 1",
[slug]
)
played_map = played_rows.each_with_object({}) { |r, h| h[r["file_path"]] = r["played_at"] }
unplayed = files.reject { |f| played_map.key?(f) }
return unplayed.first if unplayed.any?
if played_map.any?
played_map.min_by { |_path, ts| ts.to_s }[0]
else
files.first
end
return nil if rows.empty?
rows.first
end
def self.write_pls(filepath, out_path)
abs = File.absolute_path(filepath)
content = "[playlist]\nFile1=#{abs}\nTitle1=Radio Episode\nLength1=-1\nNumberOfEntries=1\nVersion=2\n"
# Build the annotated URI line station.liq consumes.
# Archived -> local file path; Live -> remote enclosure URL.
def self.annotated_uri(ep)
uri = ep["file_path"] || ep["enclosure_url"]
return nil if uri.nil? || uri.to_s.empty?
rl = ep["runlength"].to_i
title = ep["title"].to_s.gsub('"', "'")
"annotate:liq_runlength=\"#{rl}\",liq_title=\"#{title}\":#{uri}"
end
def self.write_queue_line(slug, line, out_path)
FileUtils.mkdir_p(File.dirname(out_path))
File.write(out_path, content)
File.open(out_path, "w") { |f| f.puts(line) }
end
def self.mark_as_played(slug, filepath, played_db)
played_db.execute(
"UPDATE episodes SET played_at = datetime('now') WHERE show_slug = ? AND file_path = ?",
[slug, filepath]
def self.mark_as_played(slug, guid, played_db)
jdb_exec(
played_db,
"UPDATE episodes SET played = 1, played_at = datetime('now') WHERE show_slug = ? AND guid = ?",
[slug, guid]
)
end
def self.update_all
subs_db = open_subs_db
played_db = open_played_db
shows = subs_db.query_all("SELECT slug, name FROM shows ORDER BY name")
shows = jdb_query(subs_db, "SELECT slug, name, archived FROM shows ORDER BY name")
queued = 0
skipped = 0
shows.each do |show|
slug = show["slug"]
selected = select_unplayed_episode(slug, played_db)
if selected.nil?
log_info("#{slug}: no audio files found, skipping.")
ep = select_unplayed_episode(slug, played_db)
if ep.nil?
log_info("#{slug}: no unplayed episodes, skipping.")
skipped += 1
next
end
out_pls = File.join(PLAYLISTS_DIR, "#{slug}.pls")
write_pls(selected, out_pls)
mark_as_played(slug, selected, played_db)
log_info("#{slug}: queued #{File.basename(selected)}")
line = annotated_uri(ep)
if line.nil?
log_info("#{slug}: episode #{ep['guid']} has no usable URI, skipping.")
skipped += 1
next
end
out_pls = File.join(PLAYLISTS_DIR, "#{slug}.txt")
write_queue_line(slug, line, out_pls)
mark_as_played(slug, ep["guid"], played_db)
kind = ep["file_path"] ? "downloaded" : "live"
log_info("#{slug}: queued #{ep['title']} [#{kind}] runlength=#{ep['runlength'].to_i}s")
queued += 1
end
subs_db.close
played_db.close
log_info("=== Update complete: #{queued} queued, #{skipped} skipped ===")
end
def self.json_summary
subs_db = open_subs_db
played_db = open_played_db
shows = subs_db.query_all("SELECT slug FROM shows ORDER BY name")
shows = jdb_query(subs_db, "SELECT slug FROM shows ORDER BY name")
summary = {}
shows.each do |show|
slug = show["slug"]
files = find_audio_files(File.join(PODCASTS_DIR, slug))
played_count = played_db.get_first_value(
"SELECT COUNT(*) FROM episodes WHERE show_slug = ? AND played_at IS NOT NULL", slug
)
summary[slug] = { "total_files" => files.size, "played_count" => played_count }
total = jdb_query(played_db, "SELECT COUNT(*) AS c FROM episodes WHERE show_slug = ?", [slug]).first["c"].to_i
played = jdb_query(played_db, "SELECT COUNT(*) AS c FROM episodes WHERE show_slug = ? AND played = 1", [slug]).first["c"].to_i
summary[slug] = { "total_episodes" => total, "played_count" => played, "unplayed" => total - played }
end
subs_db.close
played_db.close