mirror of
https://github.com/mistergibson/radio.git
synced 2026-09-08 22:09:51 -07:00
JRuby Installer fixes
This commit is contained in:
parent
8abce95f7d
commit
52f4c12cda
3 changed files with 139 additions and 127 deletions
|
|
@ -11,15 +11,19 @@
|
|||
# Politeness: audio/video verdict cached in shows.media_class; gpodder OPML
|
||||
# pull uses bounded retry with exponential backoff + jitter.
|
||||
|
||||
require 'sequel'
|
||||
require 'net/http'
|
||||
require 'openssl'
|
||||
require 'json'
|
||||
require 'logger'
|
||||
require 'digest/md5'
|
||||
require 'time'
|
||||
require "sequel"
|
||||
require "net/http"
|
||||
require "openssl"
|
||||
require "json"
|
||||
require "logger"
|
||||
require "time"
|
||||
require "fileutils"
|
||||
require "cgi"
|
||||
require "base64"
|
||||
require "securerandom"
|
||||
require "rexml/document"
|
||||
|
||||
ROOT = File.expand_path(File.dirname(__file__))
|
||||
ROOT = File.expand_path(File.dirname(__FILE__))
|
||||
CONFIG_PATH = File.join(ROOT, "config.json")
|
||||
|
||||
AUDIO_EXTS = [".mp3", ".m4a"].freeze
|
||||
|
|
@ -57,8 +61,6 @@ def log_error(msg)
|
|||
end
|
||||
|
||||
def setup_logging!
|
||||
fh = Logger.new(File.join($logs_dir, "fetch.log"))
|
||||
fh.formatter = $log.formatter
|
||||
$log.instance_variable_set(:@logdev,
|
||||
Logger::LogDevice.new([STDOUT, File.join($logs_dir, "fetch.log")]))
|
||||
end
|
||||
|
|
@ -106,7 +108,7 @@ SHOWS_COLUMNS = {
|
|||
opml_import: { type: :integer, default: 0 },
|
||||
archived: { type: :integer, default: 1 },
|
||||
media_class: { type: :string },
|
||||
created_at: { type: :string, default: Sequel.function(:datetime, "'now'") }
|
||||
created_at: { type: :string }
|
||||
}.freeze
|
||||
|
||||
EPISODES_COLUMNS = {
|
||||
|
|
@ -121,20 +123,24 @@ EPISODES_COLUMNS = {
|
|||
played_at: { type: :string }
|
||||
}.freeze
|
||||
|
||||
def tune(db)
|
||||
# Plain-SQL pragmas via Database#execute, which works on CRuby and JRuby/JDBC
|
||||
# across all modern Sequel versions (the :pragma extension is CRuby-only and
|
||||
# db.sql requires Sequel >= 5.42).
|
||||
db.execute("PRAGMA journal_mode=WAL;")
|
||||
db.execute("PRAGMA busy_timeout=5000;")
|
||||
end
|
||||
|
||||
def connect_subs
|
||||
db = Sequel.connect("jdbc:sqlite:#{$subs_db_path}")
|
||||
db.extension :pragma
|
||||
db.pragma journal_mode: :wal
|
||||
db.pragma busy_timeout: 5000
|
||||
tune(db)
|
||||
ensure_schema!(db, :shows, SHOWS_COLUMNS)
|
||||
db
|
||||
end
|
||||
|
||||
def connect_played
|
||||
db = Sequel.connect("jdbc:sqlite:#{$played_db_path}")
|
||||
db.extension :pragma
|
||||
db.pragma journal_mode: :wal
|
||||
db.pragma busy_timeout: 5000
|
||||
tune(db)
|
||||
ensure_schema!(db, :episodes, EPISODES_COLUMNS)
|
||||
unless db.index_exists?(:episodes, [:show_slug, :played])
|
||||
db.create_index :episodes, [:show_slug, :played], name: :idx_episodes_show_played
|
||||
|
|
@ -169,7 +175,6 @@ def slugify(name)
|
|||
end
|
||||
|
||||
def gen_uuid
|
||||
require "securerandom"
|
||||
SecureRandom.uuid
|
||||
end
|
||||
|
||||
|
|
@ -197,10 +202,8 @@ def set_media_class(db, slug, cls)
|
|||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Feed parsing (uses open-uri / rss-lite approach via Net::HTTP + REXML)
|
||||
# Feed parsing (Net::HTTP + REXML)
|
||||
# ---------------------------------------------------------------------------
|
||||
require "rexml/document"
|
||||
|
||||
def fetch_feed(feed_url)
|
||||
uri = URI.parse(feed_url)
|
||||
http = Net::HTTP.new(uri.host, uri.port)
|
||||
|
|
@ -221,7 +224,6 @@ def fetch_feed(feed_url)
|
|||
guid_el = item.elements["guid"]
|
||||
dur_el = item.elements["media:duration"] || item.elements["itunes:duration"]
|
||||
enc_el = item.elements["enclosure"]
|
||||
iso_dur_el = item.elements["itunes:duration"]
|
||||
|
||||
enclosures = []
|
||||
if enc_el
|
||||
|
|
@ -237,8 +239,7 @@ def fetch_feed(feed_url)
|
|||
title: title_el&.text,
|
||||
guid: guid_el&.text,
|
||||
enclosures: enclosures,
|
||||
duration: dur_el&.text,
|
||||
iso_duration: iso_dur_el&.text
|
||||
duration: dur_el&.text
|
||||
}
|
||||
end
|
||||
|
||||
|
|
@ -386,17 +387,7 @@ def extract_duration(entry)
|
|||
dur = entry[:duration]
|
||||
if dur
|
||||
return dur.to_i if dur.match?(/\A\d+\z/)
|
||||
m = dur.match(/\APT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?\z/)
|
||||
if m
|
||||
h = m[1] ? m[1].to_i : 0
|
||||
mn = m[2] ? m[2].to_i : 0
|
||||
s = m[3] ? m[3].to_i : 0
|
||||
return h * 3600 + mn * 60 + s
|
||||
end
|
||||
end
|
||||
iso = entry[:iso_duration]
|
||||
if iso
|
||||
m = iso.match(/\APT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?\z/)
|
||||
m = dur.match(/\APT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?\z/i)
|
||||
if m
|
||||
h = m[1] ? m[1].to_i : 0
|
||||
mn = m[2] ? m[2].to_i : 0
|
||||
|
|
@ -500,17 +491,9 @@ def fetch_show_episodes(slug, name, feed_url)
|
|||
filename = safe_filename(title, guid[-20..])
|
||||
file_path = download_episode(audio_url, dest_dir, filename)
|
||||
next if file_path.nil?
|
||||
played_db[:episodes].insert_or_ignore(
|
||||
show_slug: slug, guid: guid, title: title,
|
||||
file_path: file_path, enclosure_url: audio_url,
|
||||
runlength: duration, played: 0
|
||||
)
|
||||
insert_episode(played_db, slug, guid, title, file_path, audio_url, duration)
|
||||
else
|
||||
played_db[:episodes].insert_or_ignore(
|
||||
show_slug: slug, guid: guid, title: title,
|
||||
file_path: nil, enclosure_url: audio_url,
|
||||
runlength: duration, played: 0
|
||||
)
|
||||
insert_episode(played_db, slug, guid, title, nil, audio_url, duration)
|
||||
end
|
||||
new_count += 1
|
||||
kind = archived ? "downloaded" : "live"
|
||||
|
|
@ -522,6 +505,19 @@ def fetch_show_episodes(slug, name, feed_url)
|
|||
new_count
|
||||
end
|
||||
|
||||
def insert_episode(db, slug, guid, title, file_path, enclosure_url, duration)
|
||||
# INSERT OR IGNORE semantics via the UNIQUE(show_slug, guid) constraint.
|
||||
db.transaction do
|
||||
db[:episodes].insert(
|
||||
show_slug: slug, guid: guid, title: title,
|
||||
file_path: file_path, enclosure_url: enclosure_url,
|
||||
runlength: duration, played: 0
|
||||
)
|
||||
end
|
||||
rescue Sequel::UniqueConstraintViolation
|
||||
# Already recorded; ignore.
|
||||
end
|
||||
|
||||
def fetch_all_episodes
|
||||
db = connect_subs
|
||||
shows = db[:shows].order(:name).all
|
||||
|
|
@ -575,10 +571,14 @@ def add_show(feed_url)
|
|||
slug = slugify(name)
|
||||
guid = gen_uuid
|
||||
db = connect_subs
|
||||
db[:shows].insert_or_ignore(
|
||||
begin
|
||||
db[:shows].insert(
|
||||
slug: slug, guid: guid, name: name, feed_url: feed_url,
|
||||
source: "manual", opml_import: 0, archived: 1, media_class: cls
|
||||
)
|
||||
rescue Sequel::UniqueConstraintViolation
|
||||
# already present
|
||||
end
|
||||
db.disconnect
|
||||
$log.info("Added show: #{name} (#{slug}) [#{cls}]")
|
||||
fetch_show_episodes(slug, name, feed_url)
|
||||
|
|
@ -668,10 +668,6 @@ def run_fetch(config)
|
|||
fetch_all_episodes
|
||||
end
|
||||
|
||||
require "fileutils"
|
||||
require "cgi"
|
||||
require "base64"
|
||||
|
||||
def main
|
||||
args = ARGV.dup
|
||||
option = args.shift
|
||||
|
|
@ -698,15 +694,14 @@ def main
|
|||
when "--import-opml"
|
||||
import_opml(args.first)
|
||||
else
|
||||
needs_lock = true
|
||||
if needs_lock && !acquire_lock!
|
||||
if !acquire_lock!
|
||||
$log.info("Another radio process holds the lock; skipping this run.")
|
||||
return
|
||||
end
|
||||
begin
|
||||
run_fetch(config)
|
||||
ensure
|
||||
release_lock! if needs_lock
|
||||
release_lock!
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -104,7 +104,6 @@ echo "==> Installing gems one at a time into ${GEMS_DIR} ..."
|
|||
"$JRUBY_BIN" -S gem install --no-document sequel
|
||||
"$JRUBY_BIN" -S gem install --no-document json
|
||||
|
||||
# Give the service user ownership of the gem cache so cron runs work.
|
||||
chown -R "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "$GEMS_DIR"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -207,8 +206,9 @@ chown "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "$CONFIG_JSON"
|
|||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 9. Initialize BOTH SQLite databases with the full current schema.
|
||||
# Uses Sequel over jdbc-sqlite3, matching how the Ruby scripts access the
|
||||
# DB at runtime. Idempotent via IF NOT EXISTS semantics.
|
||||
# Uses raw CREATE TABLE IF NOT EXISTS via Database#execute - avoids the
|
||||
# Sequel create_table DSL, which is unreliable under JRuby (instance_exec'd
|
||||
# generator methods can resolve to nil). Works on CRuby and JDBC alike.
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "==> Initializing SQLite databases via Sequel/jdbc-sqlite3 ..."
|
||||
GEM_HOME="$GEMS_DIR" GEM_PATH="$GEMS_DIR" "$JRUBY_BIN" -e '
|
||||
|
|
@ -218,46 +218,53 @@ state_dir = ARGV[0]
|
|||
subs_path = File.join(state_dir, "subscriptions.db")
|
||||
played_path = File.join(state_dir, "played.db")
|
||||
|
||||
def tune(db)
|
||||
db.execute("PRAGMA journal_mode=WAL;")
|
||||
db.execute("PRAGMA busy_timeout=5000;")
|
||||
end
|
||||
|
||||
SHOWS_SQL = <<~SQL
|
||||
CREATE TABLE IF NOT EXISTS shows (
|
||||
slug TEXT PRIMARY KEY,
|
||||
guid TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
feed_url TEXT NOT NULL UNIQUE,
|
||||
source TEXT DEFAULT '"'"'manual'"'"',
|
||||
opml_import INTEGER DEFAULT 0,
|
||||
archived INTEGER DEFAULT 1,
|
||||
media_class TEXT,
|
||||
created_at TEXT
|
||||
);
|
||||
SQL
|
||||
|
||||
EPISODES_SQL = <<~SQL
|
||||
CREATE TABLE IF NOT EXISTS episodes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
show_slug TEXT NOT NULL,
|
||||
guid TEXT NOT NULL,
|
||||
title TEXT,
|
||||
file_path TEXT,
|
||||
enclosure_url TEXT,
|
||||
runlength INTEGER,
|
||||
played INTEGER DEFAULT 0,
|
||||
played_at TEXT,
|
||||
UNIQUE (show_slug, guid)
|
||||
);
|
||||
SQL
|
||||
|
||||
INDEX_SQL = <<~SQL
|
||||
CREATE INDEX IF NOT EXISTS idx_episodes_show_played ON episodes (show_slug, played);
|
||||
SQL
|
||||
|
||||
db = Sequel.connect("jdbc:sqlite:#{subs_path}")
|
||||
db.extension :pragma
|
||||
db.pragma journal_mode: :wal
|
||||
db.pragma busy_timeout: 5000
|
||||
unless db.table_exists?(:shows)
|
||||
db.create_table(:shows) do |t|
|
||||
t.primary_key :slug, type: :string
|
||||
t.string :guid, null: false, unique: true
|
||||
t.string :name, null: false
|
||||
t.string :feed_url, null: false, unique: true
|
||||
t.string :source, default: "manual"
|
||||
t.integer :opml_import, default: 0
|
||||
t.integer :archived, default: 1
|
||||
t.string :media_class
|
||||
t.string :created_at
|
||||
end
|
||||
end
|
||||
tune(db)
|
||||
db.execute(SHOWS_SQL)
|
||||
db.disconnect
|
||||
|
||||
db = Sequel.connect("jdbc:sqlite:#{played_path}")
|
||||
db.extension :pragma
|
||||
db.pragma journal_mode: :wal
|
||||
db.pragma busy_timeout: 5000
|
||||
unless db.table_exists?(:episodes)
|
||||
db.create_table(:episodes) do |t|
|
||||
t.primary_key :id
|
||||
t.string :show_slug, null: false
|
||||
t.string :guid, null: false
|
||||
t.string :title
|
||||
t.string :file_path
|
||||
t.string :enclosure_url
|
||||
t.integer :runlength
|
||||
t.integer :played, default: 0
|
||||
t.string :played_at
|
||||
t.unique_constraint %i[show_slug guid]
|
||||
end
|
||||
end
|
||||
unless db.index_exists?(:episodes, [:show_slug, :played])
|
||||
db.create_index(:episodes, [:show_slug, :played], name: :idx_episodes_show_played)
|
||||
end
|
||||
tune(db)
|
||||
db.execute(EPISODES_SQL)
|
||||
db.execute(INDEX_SQL)
|
||||
db.disconnect
|
||||
|
||||
puts " subscriptions.db and played.db initialized."
|
||||
|
|
@ -296,13 +303,12 @@ exec("liquidsoap", File.expand_path("station.liq"))
|
|||
RBRUN
|
||||
chown "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "${INSTALL_DIR}/station.rb"
|
||||
|
||||
# Allow the service user to traverse the install dir.
|
||||
chmod o+x "$INSTALL_DIR"
|
||||
|
||||
systemctl daemon-reload
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 11. Cron jobs — installed into the liquidsoap user's crontab so all
|
||||
# 11. Cron jobs - installed into the liquidsoap user's crontab so all
|
||||
# database/media writes happen under the same identity as the service.
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "==> Setting up cron jobs under user '${LIQUIDSOAP_USER}' ..."
|
||||
|
|
@ -311,14 +317,12 @@ UPDATE_PREFIX="cd ${INSTALL_DIR} && GEM_HOME=${GEMS_DIR} GEM_PATH=${GEMS_DIR} ${
|
|||
FETCH_CRON="0 * * * * ${FETCH_PREFIX} >> ${STORAGE_PATH}/logs/fetch.log 2>&1"
|
||||
UPDATE_CRON="30 * * * * ${UPDATE_PREFIX} >> ${STORAGE_PATH}/logs/update.log 2>&1"
|
||||
|
||||
# Remove any stale entries from the liquidsoap user's crontab, then add ours.
|
||||
sudo -u "$LIQUIDSOAP_USER" crontab -l 2>/dev/null | grep -vF "fetch_podcasts.rb" | grep -vF "update_playlists.rb" > /tmp/cron_ls_rb.$$ || true
|
||||
echo "$FETCH_CRON" >> /tmp/cron_ls_rb.$$
|
||||
echo "$UPDATE_CRON" >> /tmp/cron_ls_rb.$$
|
||||
sudo -u "$LIQUIDSOAP_USER" crontab /tmp/cron_ls_rb.$$
|
||||
rm -f /tmp/cron_ls_rb.$$
|
||||
|
||||
# Also scrub these from root's crontab in case an earlier install put them there.
|
||||
crontab -l 2>/dev/null | grep -vF "fetch_podcasts.rb" | grep -vF "update_playlists.rb" > /tmp/cron_root_rb.$$ || true
|
||||
crontab /tmp/cron_root_rb.$$
|
||||
rm -f /tmp/cron_root_rb.$$
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ require "json"
|
|||
require "logger"
|
||||
require "time"
|
||||
|
||||
ROOT = File.expand_path(File.dirname(__file__))
|
||||
ROOT = File.expand_path(File.dirname(__FILE__))
|
||||
CONFIG_PATH = File.join(ROOT, "config.json")
|
||||
|
||||
$state_dir = nil
|
||||
|
|
@ -40,7 +40,7 @@ def init_paths!
|
|||
end
|
||||
|
||||
$log = Logger.new(STDOUT)
|
||||
$log.formatter = proc { |msg, _sev, _time, _prog| "#{Time.now} [INFO] #{msg}\n" }
|
||||
$log.formatter = proc { |msg, _severity, _time, _progname| "#{Time.now} [INFO] #{msg}\n" }
|
||||
|
||||
def log_error(msg)
|
||||
$log.error(msg)
|
||||
|
|
@ -51,6 +51,9 @@ def setup_logging!
|
|||
Logger::LogDevice.new([STDOUT, File.join($logs_dir, "update.log")]))
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Schema: single source of truth, applied idempotently via Sequel.
|
||||
# ---------------------------------------------------------------------------
|
||||
SHOWS_COLUMNS = {
|
||||
slug: { type: :string, primary_key: true },
|
||||
guid: { type: :string, null: false, unique: true },
|
||||
|
|
@ -60,7 +63,7 @@ SHOWS_COLUMNS = {
|
|||
opml_import: { type: :integer, default: 0 },
|
||||
archived: { type: :integer, default: 1 },
|
||||
media_class: { type: :string },
|
||||
created_at: { type: :string, default: Sequel.function(:datetime, "'now'") }
|
||||
created_at: { type: :string }
|
||||
}.freeze
|
||||
|
||||
EPISODES_COLUMNS = {
|
||||
|
|
@ -75,20 +78,24 @@ EPISODES_COLUMNS = {
|
|||
played_at: { type: :string }
|
||||
}.freeze
|
||||
|
||||
def tune(db)
|
||||
# Plain-SQL pragmas via Database#execute, which works on CRuby and JRuby/JDBC
|
||||
# across all modern Sequel versions (the :pragma extension is CRuby-only and
|
||||
# db.sql requires Sequel >= 5.42).
|
||||
db.execute("PRAGMA journal_mode=WAL;")
|
||||
db.execute("PRAGMA busy_timeout=5000;")
|
||||
end
|
||||
|
||||
def connect_subs
|
||||
db = Sequel.connect("jdbc:sqlite:#{$subs_db_path}")
|
||||
db.extension :pragma
|
||||
db.pragma journal_mode: :wal
|
||||
db.pragma busy_timeout: 5000
|
||||
tune(db)
|
||||
ensure_schema!(db, :shows, SHOWS_COLUMNS)
|
||||
db
|
||||
end
|
||||
|
||||
def connect_played
|
||||
db = Sequel.connect("jdbc:sqlite:#{$played_db_path}")
|
||||
db.extension :pragma
|
||||
db.pragma journal_mode: :wal
|
||||
db.pragma busy_timeout: 5000
|
||||
tune(db)
|
||||
ensure_schema!(db, :episodes, EPISODES_COLUMNS)
|
||||
unless db.index_exists?(:episodes, [:show_slug, :played])
|
||||
db.create_index :episodes, [:show_slug, :played], name: :idx_episodes_show_played
|
||||
|
|
@ -99,7 +106,9 @@ end
|
|||
def ensure_schema!(db, table, columns)
|
||||
unless db.table_exists?(table)
|
||||
db.create_table(table) do |t|
|
||||
columns.each { |col, opts| t.column(col, **opts) }
|
||||
columns.each do |col, opts|
|
||||
t.column(col, **opts)
|
||||
end
|
||||
t.unique_constraint %i[show_slug guid] if table == :episodes
|
||||
end
|
||||
return
|
||||
|
|
@ -111,6 +120,9 @@ def ensure_schema!(db, table, columns)
|
|||
end
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mutual exclusion via flock on a shared lockfile.
|
||||
# ---------------------------------------------------------------------------
|
||||
$lock_fh = nil
|
||||
|
||||
def acquire_lock!
|
||||
|
|
@ -139,6 +151,9 @@ def release_lock!
|
|||
end
|
||||
end
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Queue-file generation
|
||||
# ---------------------------------------------------------------------------
|
||||
def annotate_uri(runlength, title, uri)
|
||||
def esc(v)
|
||||
'"' + v.to_s.gsub('"', '\\"') + '"'
|
||||
|
|
@ -157,7 +172,7 @@ end
|
|||
def mark_as_played(slug, guid, played_db)
|
||||
played_db[:episodes]
|
||||
.where(show_slug: slug, guid: guid)
|
||||
.update(played: 1, played_at: Sequel.function(:datetime, "'now'"))
|
||||
.update(played: 1, played_at: Time.now.utc.strftime("%Y-%m-%d %H:%M:%S"))
|
||||
end
|
||||
|
||||
def write_queue_file(slug, ep, archived)
|
||||
|
|
@ -200,8 +215,7 @@ def json_summary
|
|||
result = {}
|
||||
shows.each do |show|
|
||||
slug = show[:slug]
|
||||
counts = played_db[:episodes].where(show_slug: slug).hash_and_count
|
||||
total = counts.values.sum
|
||||
total = played_db[:episodes].where(show_slug: slug).count
|
||||
unplayed = played_db[:episodes].where(show_slug: slug, played: 0).count
|
||||
result[slug] = {
|
||||
name: show[:name],
|
||||
|
|
@ -241,4 +255,3 @@ def main
|
|||
end
|
||||
|
||||
main if __FILE__ == $PROGRAM_NAME
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue