CascadiaDigitalCollective/radio
CascadiaDigitalCollective/radio
Watch
1
0
Fork
You've already forked radio
 0
mirror of https://github.com/mistergibson/radio.git synced 2026-09-08 22:09:51 -07:00
Code Issues Projects Releases Packages Wiki Activity

Update

Browse source
This commit is contained in:
G. Gibson 2026-09-01 21:03:09 -07:00
commit 8abce95f7d
7 changed files with 1032 additions and 866 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,6 +1,5 @@
# Secrets and local configuration # Secrets and local configuration
config.json config.json
dirs.txt
# Runtime data (should live under the storage path, not here) # Runtime data (should live under the storage path, not here)
.venv/ .venv/
state/ state/

12
check_dirs.sh
View file

@ -1,12 +0,0 @@
#!/bin/bash
STORAGE="/mnt/storage/radio"
JINGLES="no"
ANNOUNCEMENTS="no"
if find "$STORAGE/jingles" -type f \( -name "*.mp3" -o -name "*.m4a" \) 2>/dev/null | grep -q .; then
JINGLES="yes"
fi
if find "$STORAGE/announcements" -type f \( -name "*.mp3" -o -name "*.m4a" \) 2>/dev/null | grep -q .; then
ANNOUNCEMENTS="yes"
fi
echo "jingles=$JINGLES" > /srv/radio/dirs.txt
echo "announcements=$ANNOUNCEMENTS" >> /srv/radio/dirs.txt

184
fetch_podcasts.py
View file

@ -7,16 +7,17 @@ All data (podcasts, state DBs, logs, playlists) lives under the storage path
defined in config.json ("storage" key), keeping the boot drive clean. defined in config.json ("storage" key), keeping the boot drive clean.
Concurrency model: Concurrency model:
- An exclusive, non-blocking lockfile (state/fetch.lock) guarantees this - An exclusive, non-blocking lockfile (state/radio.lock) guarantees this
process and update_playlists.py never touch the databases simultaneously. process and update_playlists.py never touch the databases simultaneously.
If the updater holds the lock, this run logs a skip and exits 0. If the updater holds the lock, this run logs a skip and exits 0.
- Both databases run in WAL mode with a busy timeout as a second safety net. - Both databases run in WAL mode with a busy timeout as a second safety net.
Filters out video podcasts: only shows whose latest enclosure has an Politeness / efficiency:
audio/* MIME type are registered. - The audio-vs-video verdict for each show is cached in subscriptions.db
(column media_class). Repeat runs reuse it instead of re-fetching every
Archived shows (archived=1) have episodes downloaded to disk; non-archived feed just to re-confirm it is audio.
shows (archived=0) store only the enclosure URL for live streaming. - The gpodder OPML pull uses bounded retry with exponential backoff + jitter
so transient failures or a 429 degrade gracefully rather than hammering.
""" """
import argparse import argparse
@ -24,18 +25,31 @@ import fcntl
import json import json
import logging import logging
import os import os
import random
import re import re
import shutil import shutil
import socket import socket
import sqlite3 import sqlite3
import sys import sys
import time
import uuid import uuid
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from pathlib import Path from pathlib import Path
from urllib.parse import quote from urllib.parse import quote
# Force IPv4 resolution so we don't stall on AAAA-first lookups when the box # --- Force IPv4-only resolution ---------------------------------------------
# has no usable IPv6 route (gpodder.net publishes both A and AAAA records). # Box has no usable IPv6 route; gpodder.net publishes both A and AAAA records,
# so dual-stack getaddrinfo returns AAAA first and stalls. Constrain every
# AF_UNSPEC lookup to AF_INET before it reaches the resolver.
_original_getaddrinfo = socket.getaddrinfo
def _ipv4_only_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
if family in (0, socket.AF_UNSPEC):
family = socket.AF_INET
return _original_getaddrinfo(host, port, family, type, proto, flags)
socket.getaddrinfo = _ipv4_only_getaddrinfo
try: try:
import requests.packages.urllib3.util.connection as _urllib3_conn import requests.packages.urllib3.util.connection as _urllib3_conn
_urllib3_conn.HAS_IPV6 = False _urllib3_conn.HAS_IPV6 = False
@ -64,7 +78,6 @@ def load_config():
return json.load(f) return json.load(f)
def init_paths(): def init_paths():
"""Resolve all data paths from config.json storage key."""
global STATE_DIR, SUBS_DB, PLAYED_DB, PODCASTS_DIR, LOGS_DIR, PLAYLISTS_DIR, LOCK_FILE global STATE_DIR, SUBS_DB, PLAYED_DB, PODCASTS_DIR, LOGS_DIR, PLAYLISTS_DIR, LOCK_FILE
cfg = load_config() cfg = load_config()
storage = Path(cfg["storage"]).expanduser().resolve() storage = Path(cfg["storage"]).expanduser().resolve()
@ -79,9 +92,7 @@ def init_paths():
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s", format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[ handlers=[logging.StreamHandler(sys.stdout)],
logging.StreamHandler(sys.stdout),
],
) )
log = logging.getLogger("fetch_podcasts") log = logging.getLogger("fetch_podcasts")
@ -89,7 +100,6 @@ def log_error(msg):
log.error(msg) log.error(msg)
def _setup_logging(): def _setup_logging():
"""Add file handler once LOGS_DIR is known."""
fh = logging.FileHandler(LOGS_DIR / "fetch.log") fh = logging.FileHandler(LOGS_DIR / "fetch.log")
fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
log.addHandler(fh) log.addHandler(fh)
@ -100,8 +110,6 @@ def _setup_logging():
_lock_fd = None _lock_fd = None
def acquire_lock(): def acquire_lock():
"""Acquire an exclusive non-blocking lock. Returns True if acquired,
False if another radio process already holds it."""
global _lock_fd global _lock_fd
STATE_DIR.mkdir(parents=True, exist_ok=True) STATE_DIR.mkdir(parents=True, exist_ok=True)
fd = os.open(str(LOCK_FILE), os.O_CREAT | os.O_RDWR, 0o644) fd = os.open(str(LOCK_FILE), os.O_CREAT | os.O_RDWR, 0o644)
@ -110,7 +118,6 @@ def acquire_lock():
except OSError: except OSError:
os.close(fd) os.close(fd)
return False return False
# Record our PID for observability.
os.ftruncate(fd, 0) os.ftruncate(fd, 0)
os.write(fd, str(os.getpid()).encode()) os.write(fd, str(os.getpid()).encode())
_lock_fd = fd _lock_fd = fd
@ -136,6 +143,7 @@ SHOWS_COLUMNS = {
"source": "TEXT DEFAULT 'manual'", "source": "TEXT DEFAULT 'manual'",
"opml_import": "INTEGER DEFAULT 0", "opml_import": "INTEGER DEFAULT 0",
"archived": "INTEGER DEFAULT 1", "archived": "INTEGER DEFAULT 1",
"media_class": "TEXT",
"created_at": "TEXT DEFAULT (datetime('now'))", "created_at": "TEXT DEFAULT (datetime('now'))",
} }
@ -159,15 +167,12 @@ def _connect(db_path):
return conn return conn
def _ensure_columns(conn, table, columns): def _ensure_columns(conn, table, columns):
"""Create the table if absent, else add any missing columns."""
exists = conn.execute( exists = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,) "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,)
).fetchone() ).fetchone()
if exists is None: if exists is None:
defs = ",\n ".join(f"{name} {spec}" for name, spec in columns.items()) defs = ",\n ".join(f"{name} {spec}" for name, spec in columns.items())
extra = "" extra = "\n ,UNIQUE(show_slug, guid)" if table == "episodes" else ""
if table == "episodes":
extra = "\n ,UNIQUE(show_slug, guid)"
conn.execute(f"CREATE TABLE {table} (\n {defs}{extra}\n )") conn.execute(f"CREATE TABLE {table} (\n {defs}{extra}\n )")
else: else:
existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")} existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")}
@ -198,27 +203,44 @@ def slugify(name):
def gen_uuid(): def gen_uuid():
return str(uuid.uuid4()) return str(uuid.uuid4())
def is_audio_feed(parsed): def classify_feed(parsed):
"""Check whether the feed's latest episode has an audio enclosure. """Return 'audio', 'video', or 'unknown' based on the latest enclosure."""
Returns True if audio, False if video or unknown.
"""
if not parsed.entries: if not parsed.entries:
return True # No entries yet; let it through, will fail on fetch return "unknown"
entry = parsed.entries[0] entry = parsed.entries[0]
enclosures = entry.get("enclosures") or [] enclosures = entry.get("enclosures") or []
if not enclosures: if not enclosures:
return True # No enclosure info; assume audio return "unknown"
mime_type = (enclosures[0].get("type") or "").lower() mime_type = (enclosures[0].get("type") or "").lower()
if mime_type.startswith("audio/"): if mime_type.startswith("audio/"):
return True return "audio"
if mime_type.startswith("video/"): if mime_type.startswith("video/"):
return False return "video"
url = (enclosures[0].get("href") or "").lower() url = (enclosures[0].get("href") or "").lower()
if any(url.endswith(ext) for ext in AUDIO_EXTS): if any(url.endswith(ext) for ext in AUDIO_EXTS):
return True return "audio"
if any(url.endswith(ext) for ext in VIDEO_EXTS): if any(url.endswith(ext) for ext in VIDEO_EXTS):
return False return "video"
return True return "unknown"
def get_media_class(subs_db, slug):
row = subs_db.execute("SELECT media_class FROM shows WHERE slug = ?", (slug,)).fetchone()
return row["media_class"] if row else None
def set_media_class(subs_db, slug, cls):
subs_db.execute("UPDATE shows SET media_class = ? WHERE slug = ?", (cls, slug))
subs_db.commit()
def fetch_feed(feed_url):
try:
parsed = feedparser.parse(feed_url)
except Exception as e:
log_error(f"Feed parse error for {feed_url}: {e}")
return None
if parsed.bozo and not parsed.entries:
log_error(f"Bozo feed (no entries) for {feed_url}: {parsed.bozo_exception}")
return None
return parsed
def gpodder_sync(cfg): def gpodder_sync(cfg):
g = cfg["gpodder"] g = cfg["gpodder"]
@ -230,12 +252,30 @@ def gpodder_sync(cfg):
print(f"--- Syncing subscriptions from {base} ---") print(f"--- Syncing subscriptions from {base} ---")
print(f"Fetching subscriptions for '{username}'...") print(f"Fetching subscriptions for '{username}'...")
# Bounded retry with exponential backoff + jitter. Retries on network
# errors and on 429/5xx; does not retry on 401/404 (auth/not-found).
max_attempts = 4
backoff_base = 2.0
resp = None
for attempt in range(1, max_attempts + 1):
try:
resp = requests.get( resp = requests.get(
url, url,
auth=(username, password), auth=(username, password),
headers={"User-Agent": "radio-automation/1.0"}, headers={"User-Agent": "radio-automation/1.0"},
timeout=60, timeout=60,
) )
break
except requests.RequestException as e:
if attempt == max_attempts:
log_error(f"gPodder sync failed after {attempt} attempts: {e}")
return []
delay = backoff_base ** attempt + random.uniform(0, 1)
log.warning("gPodder request error (%s); retrying in %.1fs", e.__class__.__name__, delay)
time.sleep(delay)
if resp is None:
return []
if resp.status_code == 200: if resp.status_code == 200:
body = resp.text body = resp.text
@ -247,8 +287,10 @@ def gpodder_sync(cfg):
log_error("gPodder sync failed: 401 Unauthorized. Check username/password in config.json.") log_error("gPodder sync failed: 401 Unauthorized. Check username/password in config.json.")
elif resp.status_code == 404: elif resp.status_code == 404:
log_error("gPodder sync failed: 404 Not Found. User may not exist or has no subscriptions.") log_error("gPodder sync failed: 404 Not Found. User may not exist or has no subscriptions.")
elif resp.status_code == 400: elif resp.status_code == 429:
log_error("gPodder sync failed: 400 Bad Request.") log_error("gPodder sync throttled (429). Will retry next cycle.")
elif resp.status_code >= 500:
log_error(f"gPodder sync server error ({resp.status_code}). Will retry next cycle.")
else: else:
log_error(f"gPodder sync failed: unexpected response {resp.status_code}: {resp.text[:200]}") log_error(f"gPodder sync failed: unexpected response {resp.status_code}: {resp.text[:200]}")
return [] return []
@ -266,11 +308,7 @@ def parse_opml(xml_string):
guid = (outline.attrib.get("guid") or "").strip() guid = (outline.attrib.get("guid") or "").strip()
if not feed_url or not re.match(r"^https?://", feed_url): if not feed_url or not re.match(r"^https?://", feed_url):
continue continue
shows.append({ shows.append({"name": name, "feed_url": feed_url, "guid": guid if guid else None})
"name": name,
"feed_url": feed_url,
"guid": guid if guid else None,
})
return shows return shows
def register_remote_shows(remote_shows): def register_remote_shows(remote_shows):
@ -282,20 +320,24 @@ def register_remote_shows(remote_shows):
existing = db.execute("SELECT slug FROM shows WHERE slug = ?", (slug,)).fetchone() existing = db.execute("SELECT slug FROM shows WHERE slug = ?", (slug,)).fetchone()
if existing is not None: if existing is not None:
continue continue
# Reuse a cached classification if we somehow already know it; otherwise
# fetch once to classify. New shows are fetched here regardless.
parsed = fetch_feed(show["feed_url"]) parsed = fetch_feed(show["feed_url"])
if parsed is None: if parsed is None:
log.warning("Skipping '%s': could not fetch feed.", show["name"]) log.warning("Skipping '%s': could not fetch feed.", show["name"])
continue continue
if not is_audio_feed(parsed): cls = classify_feed(parsed)
if cls == "video":
log.info("Skipping '%s' (%s): video podcast, not audio.", show["name"], slug) log.info("Skipping '%s' (%s): video podcast, not audio.", show["name"], slug)
skipped_video += 1 skipped_video += 1
continue continue
guid = show["guid"] or gen_uuid() guid = show["guid"] or gen_uuid()
db.execute( db.execute(
"INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'gpodder', 0, 1)", "INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived, media_class) "
(slug, guid, show["name"], show["feed_url"]), "VALUES (?, ?, ?, ?, 'gpodder', 0, 1, ?)",
(slug, guid, show["name"], show["feed_url"], cls),
) )
log.info("Registered new show: %s (%s)", show["name"], slug) log.info("Registered new show: %s (%s) [%s]", show["name"], slug, cls)
added += 1 added += 1
db.commit() db.commit()
db.close() db.close()
@ -343,17 +385,6 @@ def extract_duration(entry):
pass pass
return None return None
def fetch_feed(feed_url):
try:
parsed = feedparser.parse(feed_url)
except Exception as e:
log_error(f"Feed parse error for {feed_url}: {e}")
return None
if parsed.bozo and not parsed.entries:
log_error(f"Bozo feed (no entries) for {feed_url}: {parsed.bozo_exception}")
return None
return parsed
def download_episode(url, dest_dir, filename): def download_episode(url, dest_dir, filename):
dest = dest_dir / filename dest = dest_dir / filename
if dest.exists(): if dest.exists():
@ -384,15 +415,33 @@ def show_archived(subs_db, slug):
def fetch_show_episodes(slug, name, feed_url): def fetch_show_episodes(slug, name, feed_url):
dest_dir = PODCASTS_DIR / slug dest_dir = PODCASTS_DIR / slug
dest_dir.mkdir(parents=True, exist_ok=True) dest_dir.mkdir(parents=True, exist_ok=True)
subs_db = open_subs_db()
# Respect a cached video classification without re-fetching.
cached_cls = get_media_class(subs_db, slug)
if cached_cls == "video":
subs_db.close()
return 0
parsed = fetch_feed(feed_url) parsed = fetch_feed(feed_url)
if parsed is None: if parsed is None:
subs_db.close()
return 0 return 0
# Classify and cache if we don't already have a verdict.
if cached_cls is None:
cls = classify_feed(parsed)
set_media_class(subs_db, slug, cls)
if cls == "video":
log.info("'%s' (%s) classified as video; skipping.", name, slug)
subs_db.close()
return 0
played_db = open_played_db() played_db = open_played_db()
seen = { seen = {
row["guid"] row["guid"]
for row in played_db.execute("SELECT guid FROM episodes WHERE show_slug = ?", (slug,)) for row in played_db.execute("SELECT guid FROM episodes WHERE show_slug = ?", (slug,))
} }
subs_db = open_subs_db()
archived = show_archived(subs_db, slug) archived = show_archived(subs_db, slug)
new_count = 0 new_count = 0
for entry in parsed.entries: for entry in parsed.entries:
@ -452,16 +501,17 @@ def fetch_all_episodes():
def list_shows(detail=False): def list_shows(detail=False):
db = open_subs_db() db = open_subs_db()
rows = db.execute( rows = db.execute(
"SELECT slug, name, feed_url, source, opml_import, archived FROM shows ORDER BY name" "SELECT slug, name, feed_url, source, opml_import, archived, media_class FROM shows ORDER BY name"
).fetchall() ).fetchall()
db.close() db.close()
if not rows: if not rows:
print("No shows registered.") print("No shows registered.")
return return
print(f"{'SLUG':<30} {'ARCHIVED':<10} {'SOURCE':<10} NAME") print(f"{'SLUG':<30} {'ARCHIVED':<10} {'MEDIA':<8} {'SOURCE':<10} NAME")
for r in rows: for r in rows:
arch = "yes" if r["archived"] == 1 else "no" arch = "yes" if r["archived"] == 1 else "no"
line = f"{r['slug']:<30} {arch:<10} {r['source']:<10} {r['name']}" media = r["media_class"] or "?"
line = f"{r['slug']:<30} {arch:<10} {media:<8} {r['source']:<10} {r['name']}"
if detail: if detail:
line += f"\n{'':<50} {r['feed_url']}" line += f"\n{'':<50} {r['feed_url']}"
print(line) print(line)
@ -471,7 +521,8 @@ def add_show(feed_url):
if parsed is None or not parsed.feed.get("title"): if parsed is None or not parsed.feed.get("title"):
log_error(f"Could not determine show title from {feed_url}") log_error(f"Could not determine show title from {feed_url}")
return return
if not is_audio_feed(parsed): cls = classify_feed(parsed)
if cls == "video":
log_error(f"Refusing to add '{parsed.feed['title']}': video podcast detected.") log_error(f"Refusing to add '{parsed.feed['title']}': video podcast detected.")
return return
name = parsed.feed["title"] name = parsed.feed["title"]
@ -479,12 +530,13 @@ def add_show(feed_url):
guid = gen_uuid() guid = gen_uuid()
db = open_subs_db() db = open_subs_db()
db.execute( db.execute(
"INSERT OR IGNORE INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'manual', 0, 1)", "INSERT OR IGNORE INTO shows (slug, guid, name, feed_url, source, opml_import, archived, media_class) "
(slug, guid, name, feed_url), "VALUES (?, ?, ?, ?, 'manual', 0, 1, ?)",
(slug, guid, name, feed_url, cls),
) )
db.commit() db.commit()
db.close() db.close()
log.info("Added show: %s (%s)", name, slug) log.info("Added show: %s (%s) [%s]", name, slug, cls)
fetch_show_episodes(slug, name, feed_url) fetch_show_episodes(slug, name, feed_url)
def set_archive(slug, value): def set_archive(slug, value):
@ -545,14 +597,16 @@ def import_opml(path):
if parsed is None: if parsed is None:
log.warning("OPML import: skipping '%s', could not fetch feed.", show["name"]) log.warning("OPML import: skipping '%s', could not fetch feed.", show["name"])
continue continue
if not is_audio_feed(parsed): cls = classify_feed(parsed)
if cls == "video":
log.info("OPML import: skipping '%s' (%s): video podcast.", show["name"], slug) log.info("OPML import: skipping '%s' (%s): video podcast.", show["name"], slug)
skipped_video += 1 skipped_video += 1
continue continue
guid = show["guid"] or gen_uuid() guid = show["guid"] or gen_uuid()
db.execute( db.execute(
"INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'opml', 1, 1)", "INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived, media_class) "
(slug, guid, show["name"], show["feed_url"]), "VALUES (?, ?, ?, ?, 'opml', 1, 1, ?)",
(slug, guid, show["name"], show["feed_url"], cls),
) )
added += 1 added += 1
db.commit() db.commit()
@ -591,7 +645,6 @@ def main():
config = load_config() config = load_config()
# Read-only administrative commands don't need the write lock.
needs_lock = not args.list_shows needs_lock = not args.list_shows
if needs_lock and not acquire_lock(): if needs_lock and not acquire_lock():
@ -619,4 +672,3 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() main()

915
fetch_podcasts.rb
View file

File diff suppressed because it is too large Load diff

94
install_for_jruby
View file

@ -92,7 +92,8 @@ for tool in jruby gem bundle rake irb; do
done done
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# 5. Gems (one at a time, raised heap) # 5. Gems (one at a time, raised heap). Order matters: jdbc-sqlite3 first so
# it's present when sequel loads its JDBC SQLite subadapter.
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
export GEM_HOME="$GEMS_DIR" export GEM_HOME="$GEMS_DIR"
export GEM_PATH="$GEMS_DIR" export GEM_PATH="$GEMS_DIR"
@ -100,6 +101,7 @@ export JRUBY_OPTS="-J-Xmx1g -J-Xss512k"
echo "==> Installing gems one at a time into ${GEMS_DIR} ..." echo "==> Installing gems one at a time into ${GEMS_DIR} ..."
"$JRUBY_BIN" -S gem install --no-document jdbc-sqlite3 "$JRUBY_BIN" -S gem install --no-document jdbc-sqlite3
"$JRUBY_BIN" -S gem install --no-document sequel
"$JRUBY_BIN" -S gem install --no-document json "$JRUBY_BIN" -S gem install --no-document json
# Give the service user ownership of the gem cache so cron runs work. # Give the service user ownership of the gem cache so cron runs work.
@ -205,49 +207,61 @@ chown "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "$CONFIG_JSON"
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# 9. Initialize BOTH SQLite databases with the full current schema. # 9. Initialize BOTH SQLite databases with the full current schema.
# Uses the jdbc-sqlite3 gem (SQLite JDBC driver), matching how the Ruby # Uses Sequel over jdbc-sqlite3, matching how the Ruby scripts access the
# scripts access the DB at runtime. Idempotent via IF NOT EXISTS. # DB at runtime. Idempotent via IF NOT EXISTS semantics.
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
echo "==> Initializing SQLite databases via jdbc-sqlite3 ..." echo "==> Initializing SQLite databases via Sequel/jdbc-sqlite3 ..."
GEM_HOME="$GEMS_DIR" GEM_PATH="$GEMS_DIR" "$JRUBY_BIN" -e " GEM_HOME="$GEMS_DIR" GEM_PATH="$GEMS_DIR" "$JRUBY_BIN" -e '
require 'jdbc/sqlite3' require "sequel"
require 'java'
state_dir = ARGV[0] state_dir = ARGV[0]
JDBC::Database.new(\"jdbc:sqlite:\#{state_dir}/subscriptions.db\") do |db| subs_path = File.join(state_dir, "subscriptions.db")
db.execute(%Q{ played_path = File.join(state_dir, "played.db")
CREATE TABLE IF NOT EXISTS shows (
slug TEXT PRIMARY KEY, db = Sequel.connect("jdbc:sqlite:#{subs_path}")
guid TEXT NOT NULL UNIQUE, db.extension :pragma
name TEXT NOT NULL, db.pragma journal_mode: :wal
feed_url TEXT NOT NULL UNIQUE, db.pragma busy_timeout: 5000
source TEXT DEFAULT 'manual', unless db.table_exists?(:shows)
opml_import INTEGER DEFAULT 0, db.create_table(:shows) do |t|
archived INTEGER DEFAULT 1, t.primary_key :slug, type: :string
created_at TEXT DEFAULT (datetime('now')) t.string :guid, null: false, unique: true
) t.string :name, null: false
}) t.string :feed_url, null: false, unique: true
t.string :source, default: "manual"
t.integer :opml_import, default: 0
t.integer :archived, default: 1
t.string :media_class
t.string :created_at
end end
JDBC::Database.new(\"jdbc:sqlite:\#{state_dir}/played.db\") do |db|
db.execute(%Q{
CREATE TABLE IF NOT EXISTS episodes (
id INTEGER PRIMARY KEY AUTOINCREMENT,
show_slug TEXT NOT NULL,
guid TEXT NOT NULL,
title TEXT,
file_path TEXT,
enclosure_url TEXT,
runlength INTEGER,
played INTEGER DEFAULT 0,
played_at TEXT,
UNIQUE(show_slug, guid)
)
})
db.execute(%Q{
CREATE INDEX IF NOT EXISTS idx_episodes_show_played ON episodes (show_slug, played)
})
end end
puts ' subscriptions.db and played.db initialized.' db.disconnect
" "$STORAGE_PATH/state"
db = Sequel.connect("jdbc:sqlite:#{played_path}")
db.extension :pragma
db.pragma journal_mode: :wal
db.pragma busy_timeout: 5000
unless db.table_exists?(:episodes)
db.create_table(:episodes) do |t|
t.primary_key :id
t.string :show_slug, null: false
t.string :guid, null: false
t.string :title
t.string :file_path
t.string :enclosure_url
t.integer :runlength
t.integer :played, default: 0
t.string :played_at
t.unique_constraint %i[show_slug guid]
end
end
unless db.index_exists?(:episodes, [:show_slug, :played])
db.create_index(:episodes, [:show_slug, :played], name: :idx_episodes_show_played)
end
db.disconnect
puts " subscriptions.db and played.db initialized."
' "$STORAGE_PATH/state"
chown -R "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "$STORAGE_PATH/state" chown -R "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "$STORAGE_PATH/state"
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------

41
update_playlists.py
View file

@ -3,11 +3,9 @@
update_playlists.py - Select the next unplayed episode per show and write an update_playlists.py - Select the next unplayed episode per show and write an
annotated URI queue file for station.liq to consume. annotated URI queue file for station.liq to consume.
Concurrency model mirrors fetch_podcasts.py: Concurrency model mirrors fetch_podcasts.py: shares the same exclusive
- Shares the same exclusive lockfile (state/radio.lock). If the fetcher is lockfile (state/radio.lock); skips cleanly if the fetcher holds it. Databases
still running, this run logs a skip and exits 0 instead of hitting a run in WAL mode with a busy timeout as a second safety net.
locked database.
- Databases run in WAL mode with a busy timeout as a second safety net.
Selection: for each show, pick the earliest episode with played=0, write Selection: for each show, pick the earliest episode with played=0, write
playlists/<slug>.txt as a single annotated URI line, then mark it played. playlists/<slug>.txt as a single annotated URI line, then mark it played.
@ -22,7 +20,6 @@ import logging
import os import os
import sqlite3 import sqlite3
import sys import sys
from datetime import datetime
from pathlib import Path from pathlib import Path
ROOT = Path(__file__).resolve().parent ROOT = Path(__file__).resolve().parent
@ -55,9 +52,7 @@ def init_paths():
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s", format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[ handlers=[logging.StreamHandler(sys.stdout)],
logging.StreamHandler(sys.stdout),
],
) )
log = logging.getLogger("update_playlists") log = logging.getLogger("update_playlists")
@ -69,9 +64,7 @@ def _setup_logging():
fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
log.addHandler(fh) log.addHandler(fh)
# ---------------------------------------------------------------------------
# Shared schema definitions (must match fetch_podcasts.py). # Shared schema definitions (must match fetch_podcasts.py).
# ---------------------------------------------------------------------------
SHOWS_COLUMNS = { SHOWS_COLUMNS = {
"slug": "TEXT PRIMARY KEY", "slug": "TEXT PRIMARY KEY",
"guid": "TEXT NOT NULL UNIQUE", "guid": "TEXT NOT NULL UNIQUE",
@ -80,6 +73,7 @@ SHOWS_COLUMNS = {
"source": "TEXT DEFAULT 'manual'", "source": "TEXT DEFAULT 'manual'",
"opml_import": "INTEGER DEFAULT 0", "opml_import": "INTEGER DEFAULT 0",
"archived": "INTEGER DEFAULT 1", "archived": "INTEGER DEFAULT 1",
"media_class": "TEXT",
"created_at": "TEXT DEFAULT (datetime('now'))", "created_at": "TEXT DEFAULT (datetime('now'))",
} }
@ -108,9 +102,7 @@ def _ensure_columns(conn, table, columns):
).fetchone() ).fetchone()
if exists is None: if exists is None:
defs = ",\n ".join(f"{name} {spec}" for name, spec in columns.items()) defs = ",\n ".join(f"{name} {spec}" for name, spec in columns.items())
extra = "" extra = "\n ,UNIQUE(show_slug, guid)" if table == "episodes" else ""
if table == "episodes":
extra = "\n ,UNIQUE(show_slug, guid)"
conn.execute(f"CREATE TABLE {table} (\n {defs}{extra}\n )") conn.execute(f"CREATE TABLE {table} (\n {defs}{extra}\n )")
else: else:
existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")} existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")}
@ -134,9 +126,6 @@ def open_played_db():
conn.commit() conn.commit()
return conn return conn
# ---------------------------------------------------------------------------
# Mutual exclusion (shared with fetch_podcasts.py via the same lockfile).
# ---------------------------------------------------------------------------
_lock_fd = None _lock_fd = None
def acquire_lock(): def acquire_lock():
@ -162,39 +151,28 @@ def release_lock():
finally: finally:
_lock_fd = None _lock_fd = None
# ---------------------------------------------------------------------------
# Core logic
# ---------------------------------------------------------------------------
def _annotate_uri(runlength, title, uri): def _annotate_uri(runlength, title, uri):
"""Build a liquidsoap annotated URI line. Values must be double-quoted
and separated by commas; the whole annotation precedes a colon before
the URI. Escapes embedded double quotes minimally."""
def q(v): def q(v):
return '"' + str(v).replace('"', '\\"') + '"' return '"' + str(v).replace('"', '\\"') + '"'
ann = f"annotate:liq_runlength={q(runlength)},liq_title={q(title)}:" ann = f"annotate:liq_runlength={q(runlength)},liq_title={q(title)}:"
return ann + uri return ann + uri
def select_next_episode(slug, played_db): def select_next_episode(slug, played_db):
row = played_db.execute( return played_db.execute(
"SELECT guid, title, file_path, enclosure_url, runlength " "SELECT guid, title, file_path, enclosure_url, runlength "
"FROM episodes WHERE show_slug=? AND played=0 ORDER BY id ASC LIMIT 1", "FROM episodes WHERE show_slug=? AND played=0 ORDER BY id ASC LIMIT 1",
(slug,), (slug,),
).fetchone() ).fetchone()
return row
def mark_as_played(slug, guid, played_db): def mark_as_played(slug, guid, played_db):
played_db.execute( played_db.execute(
"UPDATE episodes SET played=1, played_at=datetime('now') " "UPDATE episodes SET played=1, played_at=datetime('now') WHERE show_slug=? AND guid=?",
"WHERE show_slug=? AND guid=?",
(slug, guid), (slug, guid),
) )
played_db.commit() played_db.commit()
def write_queue_file(slug, ep, archived): def write_queue_file(slug, ep, archived):
if archived: uri = ep["file_path"] if archived else ep["enclosure_url"]
uri = ep["file_path"]
else:
uri = ep["enclosure_url"]
if not uri: if not uri:
return None return None
line = _annotate_uri(ep["runlength"], ep["title"], uri) line = _annotate_uri(ep["runlength"], ep["title"], uri)
@ -254,7 +232,6 @@ def main():
PLAYLISTS_DIR.mkdir(parents=True, exist_ok=True) PLAYLISTS_DIR.mkdir(parents=True, exist_ok=True)
_setup_logging() _setup_logging()
# --json is read-only and doesn't need the write lock.
needs_lock = not args.json needs_lock = not args.json
if needs_lock and not acquire_lock(): if needs_lock and not acquire_lock():

359
update_playlists.rb
View file

@ -1,219 +1,244 @@
#!/usr/bin/env jruby #!/usr/bin/env jruby
# frozen_string_literal: true # frozen_string_literal: true
#
# update_playlists.rb - Select the next unplayed episode per show and write an
# annotated URI queue file for station.liq to consume (JRuby/Sequel variant).
#
# Shares the same exclusive lockfile as fetch_podcasts.rb; skips cleanly if
# the fetcher holds it. Databases run in WAL mode with a busy timeout.
require "sequel"
require "json" require "json"
require "jdbc/sqlite3" require "logger"
require "fileutils" require "time"
require "optparse"
Jdbc::SQLite3.load_driver ROOT = File.expand_path(File.dirname(__file__))
module RadioAutomation
ROOT = File.expand_path("..", __dir__)
CONFIG_PATH = File.join(ROOT, "config.json") CONFIG_PATH = File.join(ROOT, "config.json")
STORAGE_DIR = nil $state_dir = nil
STATE_DIR = nil $subs_db_path = nil
SUBS_DB = nil $played_db_path = nil
PLAYED_DB = nil $podcasts_dir = nil
PODCASTS_DIR = nil $logs_dir = nil
PLAYLISTS_DIR = nil $playlists_dir = nil
LOGS_DIR = nil $lock_file = nil
def self.init_paths def load_config
cfg = JSON.parse(File.read(CONFIG_PATH)) JSON.parse(File.read(CONFIG_PATH))
@storage = File.expand_path(cfg["storage"])
self.STORAGE_DIR = @storage
self.STATE_DIR = File.join(@storage, "state")
self.SUBS_DB = File.join(STATE_DIR, "subscriptions.db")
self.PLAYED_DB = File.join(STATE_DIR, "played.db")
self.PODCASTS_DIR = File.join(@storage, "podcasts")
self.PLAYLISTS_DIR = File.join(@storage, "playlists")
self.LOGS_DIR = File.join(@storage, "logs")
end end
# --- JDBC connection helpers --------------------------------------------- def init_paths!
cfg = load_config
def self.jdb_connect(db_file) storage = File.realpath(cfg["storage"])
java.sql.DriverManager.getConnection("jdbc:sqlite:#{db_file}") $state_dir = File.join(storage, "state")
$subs_db_path = File.join($state_dir, "subscriptions.db")
$played_db_path = File.join($state_dir, "played.db")
$podcasts_dir = File.join(storage, "podcasts")
$logs_dir = File.join(storage, "logs")
$playlists_dir = File.join(storage, "playlists")
$lock_file = File.join($state_dir, "radio.lock")
end end
def self.jdb_query(conn, sql, params = []) $log = Logger.new(STDOUT)
stmt = conn.prepareStatement(sql) $log.formatter = proc { |msg, _sev, _time, _prog| "#{Time.now} [INFO] #{msg}\n" }
params.each_with_index { |p, i| stmt.setObject(i + 1, p) }
rs = stmt.executeQuery def log_error(msg)
cols = [] $log.error(msg)
meta = rs.getMetaData
(1..meta.getColumnCount).each { |i| cols << meta.getColumnName(i) }
rows = []
while rs.next
row = {}
cols.each { |c| row[c] = rs.getObject(c) }
rows << row
end
rs.close
stmt.close
rows
end end
def self.jdb_exec(conn, sql, params = []) def setup_logging!
stmt = conn.prepareStatement(sql) $log.instance_variable_set(:@logdev,
params.each_with_index { |p, i| stmt.setObject(i + 1, p) } Logger::LogDevice.new([STDOUT, File.join($logs_dir, "update.log")]))
stmt.executeUpdate
stmt.close
end end
def self.log_info(msg) SHOWS_COLUMNS = {
puts "#{Time.now.iso8601} [INFO] #{msg}" slug: { type: :string, primary_key: true },
append_log("update.log", msg) guid: { type: :string, null: false, unique: true },
end name: { type: :string, null: false },
feed_url: { type: :string, null: false, unique: true },
source: { type: :string, default: "manual" },
opml_import: { type: :integer, default: 0 },
archived: { type: :integer, default: 1 },
media_class: { type: :string },
created_at: { type: :string, default: Sequel.function(:datetime, "'now'") }
}.freeze
def self.append_log(filename, msg) EPISODES_COLUMNS = {
FileUtils.mkdir_p(LOGS_DIR) id: { type: :integer, primary_key: true, auto_increment: true },
File.open(File.join(LOGS_DIR, filename), "a") { |f| f.puts msg } show_slug: { type: :string, null: false },
rescue StandardError guid: { type: :string, null: false },
nil title: { type: :string },
end file_path: { type: :string },
enclosure_url: { type: :string },
runlength: { type: :integer },
played: { type: :integer, default: 0 },
played_at: { type: :string }
}.freeze
def self.open_subs_db def connect_subs
db = jdb_connect(SUBS_DB) db = Sequel.connect("jdbc:sqlite:#{$subs_db_path}")
jdb_exec(db, <<-SQL) db.extension :pragma
CREATE TABLE IF NOT EXISTS shows ( db.pragma journal_mode: :wal
slug TEXT PRIMARY KEY, db.pragma busy_timeout: 5000
guid TEXT NOT NULL UNIQUE, ensure_schema!(db, :shows, SHOWS_COLUMNS)
name TEXT NOT NULL,
feed_url TEXT NOT NULL UNIQUE,
source TEXT DEFAULT 'manual',
opml_import INTEGER DEFAULT 0,
archived INTEGER DEFAULT 1,
created_at TEXT DEFAULT (datetime('now'))
)
SQL
db db
end end
def self.open_played_db def connect_played
db = jdb_connect(PLAYED_DB) db = Sequel.connect("jdbc:sqlite:#{$played_db_path}")
jdb_exec(db, <<-SQL) db.extension :pragma
CREATE TABLE IF NOT EXISTS episodes ( db.pragma journal_mode: :wal
id INTEGER PRIMARY KEY AUTOINCREMENT, db.pragma busy_timeout: 5000
show_slug TEXT NOT NULL, ensure_schema!(db, :episodes, EPISODES_COLUMNS)
guid TEXT NOT NULL, unless db.index_exists?(:episodes, [:show_slug, :played])
title TEXT, db.create_index :episodes, [:show_slug, :played], name: :idx_episodes_show_played
file_path TEXT, end
enclosure_url TEXT,
runlength INTEGER,
played INTEGER DEFAULT 0,
played_at TEXT,
UNIQUE(show_slug, guid)
)
SQL
db db
end end
# Pick the next unplayed episode for a show, keyed by guid. def ensure_schema!(db, table, columns)
# Archived: prefer a local file_path; Live: use enclosure_url. unless db.table_exists?(table)
def self.select_unplayed_episode(slug, played_db) db.create_table(table) do |t|
rows = jdb_query( columns.each { |col, opts| t.column(col, **opts) }
played_db, t.unique_constraint %i[show_slug guid] if table == :episodes
"SELECT guid, title, file_path, enclosure_url, runlength FROM episodes WHERE show_slug = ? AND played = 0 ORDER BY id ASC LIMIT 1", end
[slug] return
) end
return nil if rows.empty? existing = db.columns(table)
rows.first columns.each do |col, opts|
next if existing.include?(col)
db.alter_table(table) { |t| t.add_column(col, **opts) }
end
end end
# Build the annotated URI line station.liq consumes. $lock_fh = nil
# Archived -> local file path; Live -> remote enclosure URL.
def self.annotated_uri(ep) def acquire_lock!
uri = ep["file_path"] || ep["enclosure_url"] Dir.mkdir($state_dir) unless Dir.exist?($state_dir)
return nil if uri.nil? || uri.to_s.empty? fh = File.open($lock_file, File::RDWR | File::CREAT, 0o644)
rl = ep["runlength"].to_i begin
title = ep["title"].to_s.gsub('"', "'") fh.flock(File::LOCK_EX | File::LOCK_NB)
"annotate:liq_runlength=\"#{rl}\",liq_title=\"#{title}\":#{uri}" rescue Errno::EACCES, Errno::EAGAIN
fh.close
return false
end
fh.truncate(0)
fh.write(Process.pid.to_s)
fh.rewind
$lock_fh = fh
true
end end
def self.write_queue_line(slug, line, out_path) def release_lock!
FileUtils.mkdir_p(File.dirname(out_path)) return if $lock_fh.nil?
File.open(out_path, "w") { |f| f.puts(line) } begin
$lock_fh.flock(File::LOCK_UN)
$lock_fh.close
ensure
$lock_fh = nil
end
end end
def self.mark_as_played(slug, guid, played_db) def annotate_uri(runlength, title, uri)
jdb_exec( def esc(v)
played_db, '"' + v.to_s.gsub('"', '\\"') + '"'
"UPDATE episodes SET played = 1, played_at = datetime('now') WHERE show_slug = ? AND guid = ?", end
[slug, guid] "annotate:liq_runlength=#{esc(runlength)},liq_title=#{esc(title)}:" + uri
)
end end
def self.update_all def select_next_episode(slug, played_db)
subs_db = open_subs_db played_db[:episodes]
played_db = open_played_db .where(show_slug: slug, played: 0)
shows = jdb_query(subs_db, "SELECT slug, name, archived FROM shows ORDER BY name") .order(:id.asc)
.limit(1)
.first
end
queued = 0 def mark_as_played(slug, guid, played_db)
skipped = 0 played_db[:episodes]
.where(show_slug: slug, guid: guid)
.update(played: 1, played_at: Sequel.function(:datetime, "'now'"))
end
def write_queue_file(slug, ep, archived)
uri = archived ? ep[:file_path] : ep[:enclosure_url]
return nil if uri.nil? || uri.empty?
line = annotate_uri(ep[:runlength], ep[:title], uri)
out = File.join($playlists_dir, "#{slug}.txt")
File.write(out, line + "\n")
out
end
def update_all
subs_db = connect_subs
played_db = connect_played
shows = subs_db[:shows].order(:name).all
updated = 0
shows.each do |show| shows.each do |show|
slug = show["slug"] slug = show[:slug]
ep = select_unplayed_episode(slug, played_db) archived = show[:archived] == 1
if ep.nil? ep = select_next_episode(slug, played_db)
log_info("#{slug}: no unplayed episodes, skipping.") next if ep.nil?
skipped += 1 out = write_queue_file(slug, ep, archived)
if out.nil?
$log.warn("No playable URI for #{show[:name]} (#{slug}); skipping.")
next next
end end
line = annotated_uri(ep) mark_as_played(slug, ep[:guid], played_db)
if line.nil? updated += 1
log_info("#{slug}: episode #{ep['guid']} has no usable URI, skipping.") $log.info("Queued #{slug}: #{ep[:title]} -> #{File.basename(out)}")
skipped += 1
next
end end
out_pls = File.join(PLAYLISTS_DIR, "#{slug}.txt") subs_db.disconnect
write_queue_line(slug, line, out_pls) played_db.disconnect
mark_as_played(slug, ep["guid"], played_db) $log.info("=== Update complete: #{updated} show(s) queued ===")
kind = ep["file_path"] ? "downloaded" : "live"
log_info("#{slug}: queued #{ep['title']} [#{kind}] runlength=#{ep['runlength'].to_i}s")
queued += 1
end end
subs_db.close def json_summary
played_db.close subs_db = connect_subs
log_info("=== Update complete: #{queued} queued, #{skipped} skipped ===") played_db = connect_played
end shows = subs_db[:shows].order(:name).all
result = {}
def self.json_summary
subs_db = open_subs_db
played_db = open_played_db
shows = jdb_query(subs_db, "SELECT slug FROM shows ORDER BY name")
summary = {}
shows.each do |show| shows.each do |show|
slug = show["slug"] slug = show[:slug]
total = jdb_query(played_db, "SELECT COUNT(*) AS c FROM episodes WHERE show_slug = ?", [slug]).first["c"].to_i counts = played_db[:episodes].where(show_slug: slug).hash_and_count
played = jdb_query(played_db, "SELECT COUNT(*) AS c FROM episodes WHERE show_slug = ? AND played = 1", [slug]).first["c"].to_i total = counts.values.sum
summary[slug] = { "total_episodes" => total, "played_count" => played, "unplayed" => total - played } unplayed = played_db[:episodes].where(show_slug: slug, played: 0).count
result[slug] = {
name: show[:name],
total: total,
unplayed: unplayed,
played: total - unplayed
}
end end
subs_db.close subs_db.disconnect
played_db.close played_db.disconnect
puts JSON.pretty_generate(summary) puts JSON.pretty_generate(result)
end end
def self.main def main
options = {} args = ARGV.dup
OptionParser.new do |opts| json_mode = args.delete("--json")
opts.banner = "Usage: update_playlists.rb [options]"
opts.on("--json", "Emit JSON summary and exit") { options[:json] = true }
end.parse!
init_paths init_paths!
FileUtils.mkdir_p(STATE_DIR) [$state_dir, $logs_dir, $playlists_dir].each do |dir|
FileUtils.mkdir_p(LOGS_DIR) Dir.mkdir(dir) unless Dir.exist?(dir)
FileUtils.mkdir_p(PLAYLISTS_DIR) end
setup_logging!
if options[:json] if json_mode
json_summary json_summary
else else
if !acquire_lock!
$log.info("Another radio process holds the lock; skipping this run.")
return
end
begin
update_all update_all
ensure
release_lock!
end end
end end
end end
RadioAutomation.main main if __FILE__ == $PROGRAM_NAME