CascadiaDigitalCollective/radio
CascadiaDigitalCollective/radio
Watch
1
0
Fork
You've already forked radio
 0
mirror of https://github.com/mistergibson/radio.git synced 2026-09-08 22:09:51 -07:00
Code Issues Projects Releases Packages Wiki Activity

Update

Browse source
This commit is contained in:
G. Gibson 2026-09-01 21:03:09 -07:00
commit 8abce95f7d
7 changed files with 1032 additions and 866 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,6 +1,5 @@
# Secrets and local configuration # Secrets and local configuration
config.json config.json
dirs.txt
# Runtime data (should live under the storage path, not here) # Runtime data (should live under the storage path, not here)
.venv/ .venv/
state/ state/

12
check_dirs.sh
View file

@ -1,12 +0,0 @@
#!/bin/bash
STORAGE="/mnt/storage/radio"
JINGLES="no"
ANNOUNCEMENTS="no"
if find "$STORAGE/jingles" -type f \( -name "*.mp3" -o -name "*.m4a" \) 2>/dev/null | grep -q .; then
JINGLES="yes"
fi
if find "$STORAGE/announcements" -type f \( -name "*.mp3" -o -name "*.m4a" \) 2>/dev/null | grep -q .; then
ANNOUNCEMENTS="yes"
fi
echo "jingles=$JINGLES" > /srv/radio/dirs.txt
echo "announcements=$ANNOUNCEMENTS" >> /srv/radio/dirs.txt

184
fetch_podcasts.py
View file

@ -7,16 +7,17 @@ All data (podcasts, state DBs, logs, playlists) lives under the storage path
defined in config.json ("storage" key), keeping the boot drive clean. defined in config.json ("storage" key), keeping the boot drive clean.
Concurrency model: Concurrency model:
- An exclusive, non-blocking lockfile (state/fetch.lock) guarantees this - An exclusive, non-blocking lockfile (state/radio.lock) guarantees this
process and update_playlists.py never touch the databases simultaneously. process and update_playlists.py never touch the databases simultaneously.
If the updater holds the lock, this run logs a skip and exits 0. If the updater holds the lock, this run logs a skip and exits 0.
- Both databases run in WAL mode with a busy timeout as a second safety net. - Both databases run in WAL mode with a busy timeout as a second safety net.
Filters out video podcasts: only shows whose latest enclosure has an Politeness / efficiency:
audio/* MIME type are registered. - The audio-vs-video verdict for each show is cached in subscriptions.db
(column media_class). Repeat runs reuse it instead of re-fetching every
Archived shows (archived=1) have episodes downloaded to disk; non-archived feed just to re-confirm it is audio.
shows (archived=0) store only the enclosure URL for live streaming. - The gpodder OPML pull uses bounded retry with exponential backoff + jitter
so transient failures or a 429 degrade gracefully rather than hammering.
""" """
import argparse import argparse
@ -24,18 +25,31 @@ import fcntl
import json import json
import logging import logging
import os import os
import random
import re import re
import shutil import shutil
import socket import socket
import sqlite3 import sqlite3
import sys import sys
import time
import uuid import uuid
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from pathlib import Path from pathlib import Path
from urllib.parse import quote from urllib.parse import quote
# Force IPv4 resolution so we don't stall on AAAA-first lookups when the box # --- Force IPv4-only resolution ---------------------------------------------
# has no usable IPv6 route (gpodder.net publishes both A and AAAA records). # Box has no usable IPv6 route; gpodder.net publishes both A and AAAA records,
# so dual-stack getaddrinfo returns AAAA first and stalls. Constrain every
# AF_UNSPEC lookup to AF_INET before it reaches the resolver.
_original_getaddrinfo = socket.getaddrinfo
def _ipv4_only_getaddrinfo(host, port, family=0, type=0, proto=0, flags=0):
if family in (0, socket.AF_UNSPEC):
family = socket.AF_INET
return _original_getaddrinfo(host, port, family, type, proto, flags)
socket.getaddrinfo = _ipv4_only_getaddrinfo
try: try:
import requests.packages.urllib3.util.connection as _urllib3_conn import requests.packages.urllib3.util.connection as _urllib3_conn
_urllib3_conn.HAS_IPV6 = False _urllib3_conn.HAS_IPV6 = False
@ -64,7 +78,6 @@ def load_config():
return json.load(f) return json.load(f)
def init_paths(): def init_paths():
"""Resolve all data paths from config.json storage key."""
global STATE_DIR, SUBS_DB, PLAYED_DB, PODCASTS_DIR, LOGS_DIR, PLAYLISTS_DIR, LOCK_FILE global STATE_DIR, SUBS_DB, PLAYED_DB, PODCASTS_DIR, LOGS_DIR, PLAYLISTS_DIR, LOCK_FILE
cfg = load_config() cfg = load_config()
storage = Path(cfg["storage"]).expanduser().resolve() storage = Path(cfg["storage"]).expanduser().resolve()
@ -79,9 +92,7 @@ def init_paths():
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s", format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[ handlers=[logging.StreamHandler(sys.stdout)],
logging.StreamHandler(sys.stdout),
],
) )
log = logging.getLogger("fetch_podcasts") log = logging.getLogger("fetch_podcasts")
@ -89,7 +100,6 @@ def log_error(msg):
log.error(msg) log.error(msg)
def _setup_logging(): def _setup_logging():
"""Add file handler once LOGS_DIR is known."""
fh = logging.FileHandler(LOGS_DIR / "fetch.log") fh = logging.FileHandler(LOGS_DIR / "fetch.log")
fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
log.addHandler(fh) log.addHandler(fh)
@ -100,8 +110,6 @@ def _setup_logging():
_lock_fd = None _lock_fd = None
def acquire_lock(): def acquire_lock():
"""Acquire an exclusive non-blocking lock. Returns True if acquired,
False if another radio process already holds it."""
global _lock_fd global _lock_fd
STATE_DIR.mkdir(parents=True, exist_ok=True) STATE_DIR.mkdir(parents=True, exist_ok=True)
fd = os.open(str(LOCK_FILE), os.O_CREAT | os.O_RDWR, 0o644) fd = os.open(str(LOCK_FILE), os.O_CREAT | os.O_RDWR, 0o644)
@ -110,7 +118,6 @@ def acquire_lock():
except OSError: except OSError:
os.close(fd) os.close(fd)
return False return False
# Record our PID for observability.
os.ftruncate(fd, 0) os.ftruncate(fd, 0)
os.write(fd, str(os.getpid()).encode()) os.write(fd, str(os.getpid()).encode())
_lock_fd = fd _lock_fd = fd
@ -136,6 +143,7 @@ SHOWS_COLUMNS = {
"source": "TEXT DEFAULT 'manual'", "source": "TEXT DEFAULT 'manual'",
"opml_import": "INTEGER DEFAULT 0", "opml_import": "INTEGER DEFAULT 0",
"archived": "INTEGER DEFAULT 1", "archived": "INTEGER DEFAULT 1",
"media_class": "TEXT",
"created_at": "TEXT DEFAULT (datetime('now'))", "created_at": "TEXT DEFAULT (datetime('now'))",
} }
@ -159,15 +167,12 @@ def _connect(db_path):
return conn return conn
def _ensure_columns(conn, table, columns): def _ensure_columns(conn, table, columns):
"""Create the table if absent, else add any missing columns."""
exists = conn.execute( exists = conn.execute(
"SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,) "SELECT name FROM sqlite_master WHERE type='table' AND name=?", (table,)
).fetchone() ).fetchone()
if exists is None: if exists is None:
defs = ",\n ".join(f"{name} {spec}" for name, spec in columns.items()) defs = ",\n ".join(f"{name} {spec}" for name, spec in columns.items())
extra = "" extra = "\n ,UNIQUE(show_slug, guid)" if table == "episodes" else ""
if table == "episodes":
extra = "\n ,UNIQUE(show_slug, guid)"
conn.execute(f"CREATE TABLE {table} (\n {defs}{extra}\n )") conn.execute(f"CREATE TABLE {table} (\n {defs}{extra}\n )")
else: else:
existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")} existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")}
@ -198,27 +203,44 @@ def slugify(name):
def gen_uuid(): def gen_uuid():
return str(uuid.uuid4()) return str(uuid.uuid4())
def is_audio_feed(parsed): def classify_feed(parsed):
"""Check whether the feed's latest episode has an audio enclosure. """Return 'audio', 'video', or 'unknown' based on the latest enclosure."""
Returns True if audio, False if video or unknown.
"""
if not parsed.entries: if not parsed.entries:
return True # No entries yet; let it through, will fail on fetch return "unknown"
entry = parsed.entries[0] entry = parsed.entries[0]
enclosures = entry.get("enclosures") or [] enclosures = entry.get("enclosures") or []
if not enclosures: if not enclosures:
return True # No enclosure info; assume audio return "unknown"
mime_type = (enclosures[0].get("type") or "").lower() mime_type = (enclosures[0].get("type") or "").lower()
if mime_type.startswith("audio/"): if mime_type.startswith("audio/"):
return True return "audio"
if mime_type.startswith("video/"): if mime_type.startswith("video/"):
return False return "video"
url = (enclosures[0].get("href") or "").lower() url = (enclosures[0].get("href") or "").lower()
if any(url.endswith(ext) for ext in AUDIO_EXTS): if any(url.endswith(ext) for ext in AUDIO_EXTS):
return True return "audio"
if any(url.endswith(ext) for ext in VIDEO_EXTS): if any(url.endswith(ext) for ext in VIDEO_EXTS):
return False return "video"
return True return "unknown"
def get_media_class(subs_db, slug):
row = subs_db.execute("SELECT media_class FROM shows WHERE slug = ?", (slug,)).fetchone()
return row["media_class"] if row else None
def set_media_class(subs_db, slug, cls):
subs_db.execute("UPDATE shows SET media_class = ? WHERE slug = ?", (cls, slug))
subs_db.commit()
def fetch_feed(feed_url):
try:
parsed = feedparser.parse(feed_url)
except Exception as e:
log_error(f"Feed parse error for {feed_url}: {e}")
return None
if parsed.bozo and not parsed.entries:
log_error(f"Bozo feed (no entries) for {feed_url}: {parsed.bozo_exception}")
return None
return parsed
def gpodder_sync(cfg): def gpodder_sync(cfg):
g = cfg["gpodder"] g = cfg["gpodder"]
@ -230,12 +252,30 @@ def gpodder_sync(cfg):
print(f"--- Syncing subscriptions from {base} ---") print(f"--- Syncing subscriptions from {base} ---")
print(f"Fetching subscriptions for '{username}'...") print(f"Fetching subscriptions for '{username}'...")
# Bounded retry with exponential backoff + jitter. Retries on network
# errors and on 429/5xx; does not retry on 401/404 (auth/not-found).
max_attempts = 4
backoff_base = 2.0
resp = None
for attempt in range(1, max_attempts + 1):
try:
resp = requests.get( resp = requests.get(
url, url,
auth=(username, password), auth=(username, password),
headers={"User-Agent": "radio-automation/1.0"}, headers={"User-Agent": "radio-automation/1.0"},
timeout=60, timeout=60,
) )
break
except requests.RequestException as e:
if attempt == max_attempts:
log_error(f"gPodder sync failed after {attempt} attempts: {e}")
return []
delay = backoff_base ** attempt + random.uniform(0, 1)
log.warning("gPodder request error (%s); retrying in %.1fs", e.__class__.__name__, delay)
time.sleep(delay)
if resp is None:
return []
if resp.status_code == 200: if resp.status_code == 200:
body = resp.text body = resp.text
@ -247,8 +287,10 @@ def gpodder_sync(cfg):
log_error("gPodder sync failed: 401 Unauthorized. Check username/password in config.json.") log_error("gPodder sync failed: 401 Unauthorized. Check username/password in config.json.")
elif resp.status_code == 404: elif resp.status_code == 404:
log_error("gPodder sync failed: 404 Not Found. User may not exist or has no subscriptions.") log_error("gPodder sync failed: 404 Not Found. User may not exist or has no subscriptions.")
elif resp.status_code == 400: elif resp.status_code == 429:
log_error("gPodder sync failed: 400 Bad Request.") log_error("gPodder sync throttled (429). Will retry next cycle.")
elif resp.status_code >= 500:
log_error(f"gPodder sync server error ({resp.status_code}). Will retry next cycle.")
else: else:
log_error(f"gPodder sync failed: unexpected response {resp.status_code}: {resp.text[:200]}") log_error(f"gPodder sync failed: unexpected response {resp.status_code}: {resp.text[:200]}")
return [] return []
@ -266,11 +308,7 @@ def parse_opml(xml_string):
guid = (outline.attrib.get("guid") or "").strip() guid = (outline.attrib.get("guid") or "").strip()
if not feed_url or not re.match(r"^https?://", feed_url): if not feed_url or not re.match(r"^https?://", feed_url):
continue continue
shows.append({ shows.append({"name": name, "feed_url": feed_url, "guid": guid if guid else None})
"name": name,
"feed_url": feed_url,
"guid": guid if guid else None,
})
return shows return shows
def register_remote_shows(remote_shows): def register_remote_shows(remote_shows):
@ -282,20 +320,24 @@ def register_remote_shows(remote_shows):
existing = db.execute("SELECT slug FROM shows WHERE slug = ?", (slug,)).fetchone() existing = db.execute("SELECT slug FROM shows WHERE slug = ?", (slug,)).fetchone()
if existing is not None: if existing is not None:
continue continue
# Reuse a cached classification if we somehow already know it; otherwise
# fetch once to classify. New shows are fetched here regardless.
parsed = fetch_feed(show["feed_url"]) parsed = fetch_feed(show["feed_url"])
if parsed is None: if parsed is None:
log.warning("Skipping '%s': could not fetch feed.", show["name"]) log.warning("Skipping '%s': could not fetch feed.", show["name"])
continue continue
if not is_audio_feed(parsed): cls = classify_feed(parsed)
if cls == "video":
log.info("Skipping '%s' (%s): video podcast, not audio.", show["name"], slug) log.info("Skipping '%s' (%s): video podcast, not audio.", show["name"], slug)
skipped_video += 1 skipped_video += 1
continue continue
guid = show["guid"] or gen_uuid() guid = show["guid"] or gen_uuid()
db.execute( db.execute(
"INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'gpodder', 0, 1)", "INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived, media_class) "
(slug, guid, show["name"], show["feed_url"]), "VALUES (?, ?, ?, ?, 'gpodder', 0, 1, ?)",
(slug, guid, show["name"], show["feed_url"], cls),
) )
log.info("Registered new show: %s (%s)", show["name"], slug) log.info("Registered new show: %s (%s) [%s]", show["name"], slug, cls)
added += 1 added += 1
db.commit() db.commit()
db.close() db.close()
@ -343,17 +385,6 @@ def extract_duration(entry):
pass pass
return None return None
def fetch_feed(feed_url):
try:
parsed = feedparser.parse(feed_url)
except Exception as e:
log_error(f"Feed parse error for {feed_url}: {e}")
return None
if parsed.bozo and not parsed.entries:
log_error(f"Bozo feed (no entries) for {feed_url}: {parsed.bozo_exception}")
return None
return parsed
def download_episode(url, dest_dir, filename): def download_episode(url, dest_dir, filename):
dest = dest_dir / filename dest = dest_dir / filename
if dest.exists(): if dest.exists():
@ -384,15 +415,33 @@ def show_archived(subs_db, slug):
def fetch_show_episodes(slug, name, feed_url): def fetch_show_episodes(slug, name, feed_url):
dest_dir = PODCASTS_DIR / slug dest_dir = PODCASTS_DIR / slug
dest_dir.mkdir(parents=True, exist_ok=True) dest_dir.mkdir(parents=True, exist_ok=True)
subs_db = open_subs_db()
# Respect a cached video classification without re-fetching.
cached_cls = get_media_class(subs_db, slug)
if cached_cls == "video":
subs_db.close()
return 0
parsed = fetch_feed(feed_url) parsed = fetch_feed(feed_url)
if parsed is None: if parsed is None:
subs_db.close()
return 0 return 0
# Classify and cache if we don't already have a verdict.
if cached_cls is None:
cls = classify_feed(parsed)
set_media_class(subs_db, slug, cls)
if cls == "video":
log.info("'%s' (%s) classified as video; skipping.", name, slug)
subs_db.close()
return 0
played_db = open_played_db() played_db = open_played_db()
seen = { seen = {
row["guid"] row["guid"]
for row in played_db.execute("SELECT guid FROM episodes WHERE show_slug = ?", (slug,)) for row in played_db.execute("SELECT guid FROM episodes WHERE show_slug = ?", (slug,))
} }
subs_db = open_subs_db()
archived = show_archived(subs_db, slug) archived = show_archived(subs_db, slug)
new_count = 0 new_count = 0
for entry in parsed.entries: for entry in parsed.entries:
@ -452,16 +501,17 @@ def fetch_all_episodes():
def list_shows(detail=False): def list_shows(detail=False):
db = open_subs_db() db = open_subs_db()
rows = db.execute( rows = db.execute(
"SELECT slug, name, feed_url, source, opml_import, archived FROM shows ORDER BY name" "SELECT slug, name, feed_url, source, opml_import, archived, media_class FROM shows ORDER BY name"
).fetchall() ).fetchall()
db.close() db.close()
if not rows: if not rows:
print("No shows registered.") print("No shows registered.")
return return
print(f"{'SLUG':<30} {'ARCHIVED':<10} {'SOURCE':<10} NAME") print(f"{'SLUG':<30} {'ARCHIVED':<10} {'MEDIA':<8} {'SOURCE':<10} NAME")
for r in rows: for r in rows:
arch = "yes" if r["archived"] == 1 else "no" arch = "yes" if r["archived"] == 1 else "no"
line = f"{r['slug']:<30} {arch:<10} {r['source']:<10} {r['name']}" media = r["media_class"] or "?"
line = f"{r['slug']:<30} {arch:<10} {media:<8} {r['source']:<10} {r['name']}"
if detail: if detail:
line += f"\n{'':<50} {r['feed_url']}" line += f"\n{'':<50} {r['feed_url']}"
print(line) print(line)
@ -471,7 +521,8 @@ def add_show(feed_url):
if parsed is None or not parsed.feed.get("title"): if parsed is None or not parsed.feed.get("title"):
log_error(f"Could not determine show title from {feed_url}") log_error(f"Could not determine show title from {feed_url}")
return return
if not is_audio_feed(parsed): cls = classify_feed(parsed)
if cls == "video":
log_error(f"Refusing to add '{parsed.feed['title']}': video podcast detected.") log_error(f"Refusing to add '{parsed.feed['title']}': video podcast detected.")
return return
name = parsed.feed["title"] name = parsed.feed["title"]
@ -479,12 +530,13 @@ def add_show(feed_url):
guid = gen_uuid() guid = gen_uuid()
db = open_subs_db() db = open_subs_db()
db.execute( db.execute(
"INSERT OR IGNORE INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'manual', 0, 1)", "INSERT OR IGNORE INTO shows (slug, guid, name, feed_url, source, opml_import, archived, media_class) "
(slug, guid, name, feed_url), "VALUES (?, ?, ?, ?, 'manual', 0, 1, ?)",
(slug, guid, name, feed_url, cls),
) )
db.commit() db.commit()
db.close() db.close()
log.info("Added show: %s (%s)", name, slug) log.info("Added show: %s (%s) [%s]", name, slug, cls)
fetch_show_episodes(slug, name, feed_url) fetch_show_episodes(slug, name, feed_url)
def set_archive(slug, value): def set_archive(slug, value):
@ -545,14 +597,16 @@ def import_opml(path):
if parsed is None: if parsed is None:
log.warning("OPML import: skipping '%s', could not fetch feed.", show["name"]) log.warning("OPML import: skipping '%s', could not fetch feed.", show["name"])
continue continue
if not is_audio_feed(parsed): cls = classify_feed(parsed)
if cls == "video":
log.info("OPML import: skipping '%s' (%s): video podcast.", show["name"], slug) log.info("OPML import: skipping '%s' (%s): video podcast.", show["name"], slug)
skipped_video += 1 skipped_video += 1
continue continue
guid = show["guid"] or gen_uuid() guid = show["guid"] or gen_uuid()
db.execute( db.execute(
"INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived) VALUES (?, ?, ?, ?, 'opml', 1, 1)", "INSERT INTO shows (slug, guid, name, feed_url, source, opml_import, archived, media_class) "
(slug, guid, show["name"], show["feed_url"]), "VALUES (?, ?, ?, ?, 'opml', 1, 1, ?)",
(slug, guid, show["name"], show["feed_url"], cls),
) )
added += 1 added += 1
db.commit() db.commit()
@ -591,7 +645,6 @@ def main():
config = load_config() config = load_config()
# Read-only administrative commands don't need the write lock.
needs_lock = not args.list_shows needs_lock = not args.list_shows
if needs_lock and not acquire_lock(): if needs_lock and not acquire_lock():
@ -619,4 +672,3 @@ def main():
if __name__ == "__main__": if __name__ == "__main__":
main() main()

975
fetch_podcasts.rb
View file

File diff suppressed because it is too large Load diff

94
install_for_jruby
View file

@ -92,7 +92,8 @@ for tool in jruby gem bundle rake irb; do
done done
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# 5. Gems (one at a time, raised heap) # 5. Gems (one at a time, raised heap). Order matters: jdbc-sqlite3 first so
# it's present when sequel loads its JDBC SQLite subadapter.
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
export GEM_HOME="$GEMS_DIR" export GEM_HOME="$GEMS_DIR"
export GEM_PATH="$GEMS_DIR" export GEM_PATH="$GEMS_DIR"
@ -100,6 +101,7 @@ export JRUBY_OPTS="-J-Xmx1g -J-Xss512k"
echo "==> Installing gems one at a time into ${GEMS_DIR} ..." echo "==> Installing gems one at a time into ${GEMS_DIR} ..."
"$JRUBY_BIN" -S gem install --no-document jdbc-sqlite3 "$JRUBY_BIN" -S gem install --no-document jdbc-sqlite3
"$JRUBY_BIN" -S gem install --no-document sequel
"$JRUBY_BIN" -S gem install --no-document json "$JRUBY_BIN" -S gem install --no-document json
# Give the service user ownership of the gem cache so cron runs work. # Give the service user ownership of the gem cache so cron runs work.
@ -205,49 +207,61 @@ chown "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "$CONFIG_JSON"
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# 9. Initialize BOTH SQLite databases with the full current schema. # 9. Initialize BOTH SQLite databases with the full current schema.
# Uses the jdbc-sqlite3 gem (SQLite JDBC driver), matching how the Ruby # Uses Sequel over jdbc-sqlite3, matching how the Ruby scripts access the
# scripts access the DB at runtime. Idempotent via IF NOT EXISTS. # DB at runtime. Idempotent via IF NOT EXISTS semantics.
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
echo "==> Initializing SQLite databases via jdbc-sqlite3 ..." echo "==> Initializing SQLite databases via Sequel/jdbc-sqlite3 ..."
GEM_HOME="$GEMS_DIR" GEM_PATH="$GEMS_DIR" "$JRUBY_BIN" -e " GEM_HOME="$GEMS_DIR" GEM_PATH="$GEMS_DIR" "$JRUBY_BIN" -e '
require 'jdbc/sqlite3' require "sequel"
require 'java'
state_dir = ARGV[0] state_dir = ARGV[0]
JDBC::Database.new(\"jdbc:sqlite:\#{state_dir}/subscriptions.db\") do |db| subs_path = File.join(state_dir, "subscriptions.db")
db.execute(%Q{ played_path = File.join(state_dir, "played.db")
CREATE TABLE IF NOT EXISTS shows (
slug TEXT PRIMARY KEY, db = Sequel.connect("jdbc:sqlite:#{subs_path}")
guid TEXT NOT NULL UNIQUE, db.extension :pragma
name TEXT NOT NULL, db.pragma journal_mode: :wal
feed_url TEXT NOT NULL UNIQUE, db.pragma busy_timeout: 5000
source TEXT DEFAULT 'manual', unless db.table_exists?(:shows)
opml_import INTEGER DEFAULT 0, db.create_table(:shows) do |t|
archived INTEGER DEFAULT 1, t.primary_key :slug, type: :string
created_at TEXT DEFAULT (datetime('now')) t.string :guid, null: false, unique: true
) t.string :name, null: false
}) t.string :feed_url, null: false, unique: true
t.string :source, default: "manual"
t.integer :opml_import, default: 0
t.integer :archived, default: 1
t.string :media_class
t.string :created_at
end
end end
JDBC::Database.new(\"jdbc:sqlite:\#{state_dir}/played.db\") do |db| db.disconnect
db.execute(%Q{
CREATE TABLE IF NOT EXISTS episodes ( db = Sequel.connect("jdbc:sqlite:#{played_path}")
id INTEGER PRIMARY KEY AUTOINCREMENT, db.extension :pragma
show_slug TEXT NOT NULL, db.pragma journal_mode: :wal
guid TEXT NOT NULL, db.pragma busy_timeout: 5000
title TEXT, unless db.table_exists?(:episodes)
file_path TEXT, db.create_table(:episodes) do |t|
enclosure_url TEXT, t.primary_key :id
runlength INTEGER, t.string :show_slug, null: false
played INTEGER DEFAULT 0, t.string :guid, null: false
played_at TEXT, t.string :title
UNIQUE(show_slug, guid) t.string :file_path
) t.string :enclosure_url
}) t.integer :runlength
db.execute(%Q{ t.integer :played, default: 0
CREATE INDEX IF NOT EXISTS idx_episodes_show_played ON episodes (show_slug, played) t.string :played_at
}) t.unique_constraint %i[show_slug guid]
end
end end
puts ' subscriptions.db and played.db initialized.' unless db.index_exists?(:episodes, [:show_slug, :played])
" "$STORAGE_PATH/state" db.create_index(:episodes, [:show_slug, :played], name: :idx_episodes_show_played)
end
db.disconnect
puts " subscriptions.db and played.db initialized."
' "$STORAGE_PATH/state"
chown -R "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "$STORAGE_PATH/state" chown -R "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "$STORAGE_PATH/state"
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------

41
update_playlists.py
View file

@ -3,11 +3,9 @@
update_playlists.py - Select the next unplayed episode per show and write an update_playlists.py - Select the next unplayed episode per show and write an
annotated URI queue file for station.liq to consume. annotated URI queue file for station.liq to consume.
Concurrency model mirrors fetch_podcasts.py: Concurrency model mirrors fetch_podcasts.py: shares the same exclusive
- Shares the same exclusive lockfile (state/radio.lock). If the fetcher is lockfile (state/radio.lock); skips cleanly if the fetcher holds it. Databases
still running, this run logs a skip and exits 0 instead of hitting a run in WAL mode with a busy timeout as a second safety net.
locked database.
- Databases run in WAL mode with a busy timeout as a second safety net.
Selection: for each show, pick the earliest episode with played=0, write Selection: for each show, pick the earliest episode with played=0, write
playlists/<slug>.txt as a single annotated URI line, then mark it played. playlists/<slug>.txt as a single annotated URI line, then mark it played.
@ -22,7 +20,6 @@ import logging
import os import os
import sqlite3 import sqlite3
import sys import sys
from datetime import datetime
from pathlib import Path from pathlib import Path
ROOT = Path(__file__).resolve().parent ROOT = Path(__file__).resolve().parent
@ -55,9 +52,7 @@ def init_paths():
logging.basicConfig( logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s", format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[ handlers=[logging.StreamHandler(sys.stdout)],
logging.StreamHandler(sys.stdout),
],
) )
log = logging.getLogger("update_playlists") log = logging.getLogger("update_playlists")
@ -69,9 +64,7 @@ def _setup_logging():
fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) fh.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
log.addHandler(fh) log.addHandler(fh)
# ---------------------------------------------------------------------------
# Shared schema definitions (must match fetch_podcasts.py). # Shared schema definitions (must match fetch_podcasts.py).
# ---------------------------------------------------------------------------
SHOWS_COLUMNS = { SHOWS_COLUMNS = {
"slug": "TEXT PRIMARY KEY", "slug": "TEXT PRIMARY KEY",
"guid": "TEXT NOT NULL UNIQUE", "guid": "TEXT NOT NULL UNIQUE",
@ -80,6 +73,7 @@ SHOWS_COLUMNS = {
"source": "TEXT DEFAULT 'manual'", "source": "TEXT DEFAULT 'manual'",
"opml_import": "INTEGER DEFAULT 0", "opml_import": "INTEGER DEFAULT 0",
"archived": "INTEGER DEFAULT 1", "archived": "INTEGER DEFAULT 1",
"media_class": "TEXT",
"created_at": "TEXT DEFAULT (datetime('now'))", "created_at": "TEXT DEFAULT (datetime('now'))",
} }
@ -108,9 +102,7 @@ def _ensure_columns(conn, table, columns):
).fetchone() ).fetchone()
if exists is None: if exists is None:
defs = ",\n ".join(f"{name} {spec}" for name, spec in columns.items()) defs = ",\n ".join(f"{name} {spec}" for name, spec in columns.items())
extra = "" extra = "\n ,UNIQUE(show_slug, guid)" if table == "episodes" else ""
if table == "episodes":
extra = "\n ,UNIQUE(show_slug, guid)"
conn.execute(f"CREATE TABLE {table} (\n {defs}{extra}\n )") conn.execute(f"CREATE TABLE {table} (\n {defs}{extra}\n )")
else: else:
existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")} existing = {row[1] for row in conn.execute(f"PRAGMA table_info({table})")}
@ -134,9 +126,6 @@ def open_played_db():
conn.commit() conn.commit()
return conn return conn
# ---------------------------------------------------------------------------
# Mutual exclusion (shared with fetch_podcasts.py via the same lockfile).
# ---------------------------------------------------------------------------
_lock_fd = None _lock_fd = None
def acquire_lock(): def acquire_lock():
@ -162,39 +151,28 @@ def release_lock():
finally: finally:
_lock_fd = None _lock_fd = None
# ---------------------------------------------------------------------------
# Core logic
# ---------------------------------------------------------------------------
def _annotate_uri(runlength, title, uri): def _annotate_uri(runlength, title, uri):
"""Build a liquidsoap annotated URI line. Values must be double-quoted
and separated by commas; the whole annotation precedes a colon before
the URI. Escapes embedded double quotes minimally."""
def q(v): def q(v):
return '"' + str(v).replace('"', '\\"') + '"' return '"' + str(v).replace('"', '\\"') + '"'
ann = f"annotate:liq_runlength={q(runlength)},liq_title={q(title)}:" ann = f"annotate:liq_runlength={q(runlength)},liq_title={q(title)}:"
return ann + uri return ann + uri
def select_next_episode(slug, played_db): def select_next_episode(slug, played_db):
row = played_db.execute( return played_db.execute(
"SELECT guid, title, file_path, enclosure_url, runlength " "SELECT guid, title, file_path, enclosure_url, runlength "
"FROM episodes WHERE show_slug=? AND played=0 ORDER BY id ASC LIMIT 1", "FROM episodes WHERE show_slug=? AND played=0 ORDER BY id ASC LIMIT 1",
(slug,), (slug,),
).fetchone() ).fetchone()
return row
def mark_as_played(slug, guid, played_db): def mark_as_played(slug, guid, played_db):
played_db.execute( played_db.execute(
"UPDATE episodes SET played=1, played_at=datetime('now') " "UPDATE episodes SET played=1, played_at=datetime('now') WHERE show_slug=? AND guid=?",
"WHERE show_slug=? AND guid=?",
(slug, guid), (slug, guid),
) )
played_db.commit() played_db.commit()
def write_queue_file(slug, ep, archived): def write_queue_file(slug, ep, archived):
if archived: uri = ep["file_path"] if archived else ep["enclosure_url"]
uri = ep["file_path"]
else:
uri = ep["enclosure_url"]
if not uri: if not uri:
return None return None
line = _annotate_uri(ep["runlength"], ep["title"], uri) line = _annotate_uri(ep["runlength"], ep["title"], uri)
@ -254,7 +232,6 @@ def main():
PLAYLISTS_DIR.mkdir(parents=True, exist_ok=True) PLAYLISTS_DIR.mkdir(parents=True, exist_ok=True)
_setup_logging() _setup_logging()
# --json is read-only and doesn't need the write lock.
needs_lock = not args.json needs_lock = not args.json
if needs_lock and not acquire_lock(): if needs_lock and not acquire_lock():

379
update_playlists.rb
View file

@ -1,219 +1,244 @@
#!/usr/bin/env jruby #!/usr/bin/env jruby
# frozen_string_literal: true # frozen_string_literal: true
#
# update_playlists.rb - Select the next unplayed episode per show and write an
# annotated URI queue file for station.liq to consume (JRuby/Sequel variant).
#
# Shares the same exclusive lockfile as fetch_podcasts.rb; skips cleanly if
# the fetcher holds it. Databases run in WAL mode with a busy timeout.
require "sequel"
require "json" require "json"
require "jdbc/sqlite3" require "logger"
require "fileutils" require "time"
require "optparse"
Jdbc::SQLite3.load_driver ROOT = File.expand_path(File.dirname(__file__))
CONFIG_PATH = File.join(ROOT, "config.json")
module RadioAutomation $state_dir = nil
ROOT = File.expand_path("..", __dir__) $subs_db_path = nil
CONFIG_PATH = File.join(ROOT, "config.json") $played_db_path = nil
$podcasts_dir = nil
$logs_dir = nil
$playlists_dir = nil
$lock_file = nil
STORAGE_DIR = nil def load_config
STATE_DIR = nil JSON.parse(File.read(CONFIG_PATH))
SUBS_DB = nil end
PLAYED_DB = nil
PODCASTS_DIR = nil
PLAYLISTS_DIR = nil
LOGS_DIR = nil
def self.init_paths def init_paths!
cfg = JSON.parse(File.read(CONFIG_PATH)) cfg = load_config
@storage = File.expand_path(cfg["storage"]) storage = File.realpath(cfg["storage"])
self.STORAGE_DIR = @storage $state_dir = File.join(storage, "state")
self.STATE_DIR = File.join(@storage, "state") $subs_db_path = File.join($state_dir, "subscriptions.db")
self.SUBS_DB = File.join(STATE_DIR, "subscriptions.db") $played_db_path = File.join($state_dir, "played.db")
self.PLAYED_DB = File.join(STATE_DIR, "played.db") $podcasts_dir = File.join(storage, "podcasts")
self.PODCASTS_DIR = File.join(@storage, "podcasts") $logs_dir = File.join(storage, "logs")
self.PLAYLISTS_DIR = File.join(@storage, "playlists") $playlists_dir = File.join(storage, "playlists")
self.LOGS_DIR = File.join(@storage, "logs") $lock_file = File.join($state_dir, "radio.lock")
end end
# --- JDBC connection helpers --------------------------------------------- $log = Logger.new(STDOUT)
$log.formatter = proc { |msg, _sev, _time, _prog| "#{Time.now} [INFO] #{msg}\n" }
def self.jdb_connect(db_file) def log_error(msg)
java.sql.DriverManager.getConnection("jdbc:sqlite:#{db_file}") $log.error(msg)
end end
def self.jdb_query(conn, sql, params = []) def setup_logging!
stmt = conn.prepareStatement(sql) $log.instance_variable_set(:@logdev,
params.each_with_index { |p, i| stmt.setObject(i + 1, p) } Logger::LogDevice.new([STDOUT, File.join($logs_dir, "update.log")]))
rs = stmt.executeQuery end
cols = []
meta = rs.getMetaData
(1..meta.getColumnCount).each { |i| cols << meta.getColumnName(i) }
rows = []
while rs.next
row = {}
cols.each { |c| row[c] = rs.getObject(c) }
rows << row
end
rs.close
stmt.close
rows
end
def self.jdb_exec(conn, sql, params = []) SHOWS_COLUMNS = {
stmt = conn.prepareStatement(sql) slug: { type: :string, primary_key: true },
params.each_with_index { |p, i| stmt.setObject(i + 1, p) } guid: { type: :string, null: false, unique: true },
stmt.executeUpdate name: { type: :string, null: false },
stmt.close feed_url: { type: :string, null: false, unique: true },
end source: { type: :string, default: "manual" },
opml_import: { type: :integer, default: 0 },
archived: { type: :integer, default: 1 },
media_class: { type: :string },
created_at: { type: :string, default: Sequel.function(:datetime, "'now'") }
}.freeze
def self.log_info(msg) EPISODES_COLUMNS = {
puts "#{Time.now.iso8601} [INFO] #{msg}" id: { type: :integer, primary_key: true, auto_increment: true },
append_log("update.log", msg) show_slug: { type: :string, null: false },
end guid: { type: :string, null: false },
title: { type: :string },
file_path: { type: :string },
enclosure_url: { type: :string },
runlength: { type: :integer },
played: { type: :integer, default: 0 },
played_at: { type: :string }
}.freeze
def self.append_log(filename, msg) def connect_subs
FileUtils.mkdir_p(LOGS_DIR) db = Sequel.connect("jdbc:sqlite:#{$subs_db_path}")
File.open(File.join(LOGS_DIR, filename), "a") { |f| f.puts msg } db.extension :pragma
rescue StandardError db.pragma journal_mode: :wal
nil db.pragma busy_timeout: 5000
end ensure_schema!(db, :shows, SHOWS_COLUMNS)
def self.open_subs_db
db = jdb_connect(SUBS_DB)
jdb_exec(db, <<-SQL)
CREATE TABLE IF NOT EXISTS shows (
slug TEXT PRIMARY KEY,
guid TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
feed_url TEXT NOT NULL UNIQUE,
source TEXT DEFAULT 'manual',
opml_import INTEGER DEFAULT 0,
archived INTEGER DEFAULT 1,
created_at TEXT DEFAULT (datetime('now'))
)
SQL
db db
end end
def self.open_played_db def connect_played
db = jdb_connect(PLAYED_DB) db = Sequel.connect("jdbc:sqlite:#{$played_db_path}")
jdb_exec(db, <<-SQL) db.extension :pragma
CREATE TABLE IF NOT EXISTS episodes ( db.pragma journal_mode: :wal
id INTEGER PRIMARY KEY AUTOINCREMENT, db.pragma busy_timeout: 5000
show_slug TEXT NOT NULL, ensure_schema!(db, :episodes, EPISODES_COLUMNS)
guid TEXT NOT NULL, unless db.index_exists?(:episodes, [:show_slug, :played])
title TEXT, db.create_index :episodes, [:show_slug, :played], name: :idx_episodes_show_played
file_path TEXT, end
enclosure_url TEXT,
runlength INTEGER,
played INTEGER DEFAULT 0,
played_at TEXT,
UNIQUE(show_slug, guid)
)
SQL
db db
end
def ensure_schema!(db, table, columns)
unless db.table_exists?(table)
db.create_table(table) do |t|
columns.each { |col, opts| t.column(col, **opts) }
t.unique_constraint %i[show_slug guid] if table == :episodes
end end
return
# Pick the next unplayed episode for a show, keyed by guid.
# Archived: prefer a local file_path; Live: use enclosure_url.
def self.select_unplayed_episode(slug, played_db)
rows = jdb_query(
played_db,
"SELECT guid, title, file_path, enclosure_url, runlength FROM episodes WHERE show_slug = ? AND played = 0 ORDER BY id ASC LIMIT 1",
[slug]
)
return nil if rows.empty?
rows.first
end end
existing = db.columns(table)
# Build the annotated URI line station.liq consumes. columns.each do |col, opts|
# Archived -> local file path; Live -> remote enclosure URL. next if existing.include?(col)
def self.annotated_uri(ep) db.alter_table(table) { |t| t.add_column(col, **opts) }
uri = ep["file_path"] || ep["enclosure_url"]
return nil if uri.nil? || uri.to_s.empty?
rl = ep["runlength"].to_i
title = ep["title"].to_s.gsub('"', "'")
"annotate:liq_runlength=\"#{rl}\",liq_title=\"#{title}\":#{uri}"
end end
end
def self.write_queue_line(slug, line, out_path) $lock_fh = nil
FileUtils.mkdir_p(File.dirname(out_path))
File.open(out_path, "w") { |f| f.puts(line) } def acquire_lock!
Dir.mkdir($state_dir) unless Dir.exist?($state_dir)
fh = File.open($lock_file, File::RDWR | File::CREAT, 0o644)
begin
fh.flock(File::LOCK_EX | File::LOCK_NB)
rescue Errno::EACCES, Errno::EAGAIN
fh.close
return false
end end
fh.truncate(0)
fh.write(Process.pid.to_s)
fh.rewind
$lock_fh = fh
true
end
def self.mark_as_played(slug, guid, played_db) def release_lock!
jdb_exec( return if $lock_fh.nil?
played_db, begin
"UPDATE episodes SET played = 1, played_at = datetime('now') WHERE show_slug = ? AND guid = ?", $lock_fh.flock(File::LOCK_UN)
[slug, guid] $lock_fh.close
) ensure
$lock_fh = nil
end end
end
def self.update_all def annotate_uri(runlength, title, uri)
subs_db = open_subs_db def esc(v)
played_db = open_played_db '"' + v.to_s.gsub('"', '\\"') + '"'
shows = jdb_query(subs_db, "SELECT slug, name, archived FROM shows ORDER BY name") end
"annotate:liq_runlength=#{esc(runlength)},liq_title=#{esc(title)}:" + uri
end
queued = 0 def select_next_episode(slug, played_db)
skipped = 0 played_db[:episodes]
.where(show_slug: slug, played: 0)
.order(:id.asc)
.limit(1)
.first
end
def mark_as_played(slug, guid, played_db)
played_db[:episodes]
.where(show_slug: slug, guid: guid)
.update(played: 1, played_at: Sequel.function(:datetime, "'now'"))
end
def write_queue_file(slug, ep, archived)
uri = archived ? ep[:file_path] : ep[:enclosure_url]
return nil if uri.nil? || uri.empty?
line = annotate_uri(ep[:runlength], ep[:title], uri)
out = File.join($playlists_dir, "#{slug}.txt")
File.write(out, line + "\n")
out
end
def update_all
subs_db = connect_subs
played_db = connect_played
shows = subs_db[:shows].order(:name).all
updated = 0
shows.each do |show| shows.each do |show|
slug = show["slug"] slug = show[:slug]
ep = select_unplayed_episode(slug, played_db) archived = show[:archived] == 1
if ep.nil? ep = select_next_episode(slug, played_db)
log_info("#{slug}: no unplayed episodes, skipping.") next if ep.nil?
skipped += 1 out = write_queue_file(slug, ep, archived)
if out.nil?
$log.warn("No playable URI for #{show[:name]} (#{slug}); skipping.")
next next
end end
line = annotated_uri(ep) mark_as_played(slug, ep[:guid], played_db)
if line.nil? updated += 1
log_info("#{slug}: episode #{ep['guid']} has no usable URI, skipping.") $log.info("Queued #{slug}: #{ep[:title]} -> #{File.basename(out)}")
skipped += 1
next
end
out_pls = File.join(PLAYLISTS_DIR, "#{slug}.txt")
write_queue_line(slug, line, out_pls)
mark_as_played(slug, ep["guid"], played_db)
kind = ep["file_path"] ? "downloaded" : "live"
log_info("#{slug}: queued #{ep['title']} [#{kind}] runlength=#{ep['runlength'].to_i}s")
queued += 1
end end
subs_db.disconnect
played_db.disconnect
$log.info("=== Update complete: #{updated} show(s) queued ===")
end
subs_db.close def json_summary
played_db.close subs_db = connect_subs
log_info("=== Update complete: #{queued} queued, #{skipped} skipped ===") played_db = connect_played
end shows = subs_db[:shows].order(:name).all
result = {}
def self.json_summary
subs_db = open_subs_db
played_db = open_played_db
shows = jdb_query(subs_db, "SELECT slug FROM shows ORDER BY name")
summary = {}
shows.each do |show| shows.each do |show|
slug = show["slug"] slug = show[:slug]
total = jdb_query(played_db, "SELECT COUNT(*) AS c FROM episodes WHERE show_slug = ?", [slug]).first["c"].to_i counts = played_db[:episodes].where(show_slug: slug).hash_and_count
played = jdb_query(played_db, "SELECT COUNT(*) AS c FROM episodes WHERE show_slug = ? AND played = 1", [slug]).first["c"].to_i total = counts.values.sum
summary[slug] = { "total_episodes" => total, "played_count" => played, "unplayed" => total - played } unplayed = played_db[:episodes].where(show_slug: slug, played: 0).count
result[slug] = {
name: show[:name],
total: total,
unplayed: unplayed,
played: total - unplayed
}
end end
subs_db.close subs_db.disconnect
played_db.close played_db.disconnect
puts JSON.pretty_generate(summary) puts JSON.pretty_generate(result)
end
def main
args = ARGV.dup
json_mode = args.delete("--json")
init_paths!
[$state_dir, $logs_dir, $playlists_dir].each do |dir|
Dir.mkdir(dir) unless Dir.exist?(dir)
end end
setup_logging!
def self.main if json_mode
options = {}
OptionParser.new do |opts|
opts.banner = "Usage: update_playlists.rb [options]"
opts.on("--json", "Emit JSON summary and exit") { options[:json] = true }
end.parse!
init_paths
FileUtils.mkdir_p(STATE_DIR)
FileUtils.mkdir_p(LOGS_DIR)
FileUtils.mkdir_p(PLAYLISTS_DIR)
if options[:json]
json_summary json_summary
else else
if !acquire_lock!
$log.info("Another radio process holds the lock; skipping this run.")
return
end
begin
update_all update_all
ensure
release_lock!
end end
end end
end end
RadioAutomation.main main if __FILE__ == $PROGRAM_NAME