diff --git a/fetch_podcasts.rb b/fetch_podcasts.rb index d86ad8d..b3b9fd4 100755 --- a/fetch_podcasts.rb +++ b/fetch_podcasts.rb @@ -11,15 +11,19 @@ # Politeness: audio/video verdict cached in shows.media_class; gpodder OPML # pull uses bounded retry with exponential backoff + jitter. -require 'sequel' -require 'net/http' -require 'openssl' -require 'json' -require 'logger' -require 'digest/md5' -require 'time' +require "sequel" +require "net/http" +require "openssl" +require "json" +require "logger" +require "time" +require "fileutils" +require "cgi" +require "base64" +require "securerandom" +require "rexml/document" -ROOT = File.expand_path(File.dirname(__file__)) +ROOT = File.expand_path(File.dirname(__FILE__)) CONFIG_PATH = File.join(ROOT, "config.json") AUDIO_EXTS = [".mp3", ".m4a"].freeze @@ -57,8 +61,6 @@ def log_error(msg) end def setup_logging! - fh = Logger.new(File.join($logs_dir, "fetch.log")) - fh.formatter = $log.formatter $log.instance_variable_set(:@logdev, Logger::LogDevice.new([STDOUT, File.join($logs_dir, "fetch.log")])) end @@ -106,7 +108,7 @@ SHOWS_COLUMNS = { opml_import: { type: :integer, default: 0 }, archived: { type: :integer, default: 1 }, media_class: { type: :string }, - created_at: { type: :string, default: Sequel.function(:datetime, "'now'") } + created_at: { type: :string } }.freeze EPISODES_COLUMNS = { @@ -121,20 +123,24 @@ EPISODES_COLUMNS = { played_at: { type: :string } }.freeze +def tune(db) + # Plain-SQL pragmas via Database#execute, which works on CRuby and JRuby/JDBC + # across all modern Sequel versions (the :pragma extension is CRuby-only and + # db.sql requires Sequel >= 5.42). + db.execute("PRAGMA journal_mode=WAL;") + db.execute("PRAGMA busy_timeout=5000;") +end + def connect_subs db = Sequel.connect("jdbc:sqlite:#{$subs_db_path}") - db.extension :pragma - db.pragma journal_mode: :wal - db.pragma busy_timeout: 5000 + tune(db) ensure_schema!(db, :shows, SHOWS_COLUMNS) db end def connect_played db = Sequel.connect("jdbc:sqlite:#{$played_db_path}") - db.extension :pragma - db.pragma journal_mode: :wal - db.pragma busy_timeout: 5000 + tune(db) ensure_schema!(db, :episodes, EPISODES_COLUMNS) unless db.index_exists?(:episodes, [:show_slug, :played]) db.create_index :episodes, [:show_slug, :played], name: :idx_episodes_show_played @@ -169,7 +175,6 @@ def slugify(name) end def gen_uuid - require "securerandom" SecureRandom.uuid end @@ -197,10 +202,8 @@ def set_media_class(db, slug, cls) end # --------------------------------------------------------------------------- -# Feed parsing (uses open-uri / rss-lite approach via Net::HTTP + REXML) +# Feed parsing (Net::HTTP + REXML) # --------------------------------------------------------------------------- -require "rexml/document" - def fetch_feed(feed_url) uri = URI.parse(feed_url) http = Net::HTTP.new(uri.host, uri.port) @@ -216,29 +219,27 @@ def fetch_feed(feed_url) title = channel.elements["title"]&.text entries = [] channel.get_elements("./item").each do |item| - link_el = item.elements["link"] + link_el = item.elements["link"] title_el = item.elements["title"] - guid_el = item.elements["guid"] - dur_el = item.elements["media:duration"] || item.elements["itunes:duration"] - enc_el = item.elements["enclosure"] - iso_dur_el = item.elements["itunes:duration"] + guid_el = item.elements["guid"] + dur_el = item.elements["media:duration"] || item.elements["itunes:duration"] + enc_el = item.elements["enclosure"] enclosures = [] if enc_el enclosures << { - href: enc_el.attributes["url"], - type: enc_el.attributes["type"], + href: enc_el.attributes["url"], + type: enc_el.attributes["type"], length: enc_el.attributes["length"] } end entries << { - link: link_el&.text, - title: title_el&.text, - guid: guid_el&.text, + link: link_el&.text, + title: title_el&.text, + guid: guid_el&.text, enclosures: enclosures, - duration: dur_el&.text, - iso_duration: iso_dur_el&.text + duration: dur_el&.text } end @@ -386,21 +387,11 @@ def extract_duration(entry) dur = entry[:duration] if dur return dur.to_i if dur.match?(/\A\d+\z/) - m = dur.match(/\APT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?\z/) + m = dur.match(/\APT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?\z/i) if m - h = m[1] ? m[1].to_i : 0 + h = m[1] ? m[1].to_i : 0 mn = m[2] ? m[2].to_i : 0 - s = m[3] ? m[3].to_i : 0 - return h * 3600 + mn * 60 + s - end - end - iso = entry[:iso_duration] - if iso - m = iso.match(/\APT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?\z/) - if m - h = m[1] ? m[1].to_i : 0 - mn = m[2] ? m[2].to_i : 0 - s = m[3] ? m[3].to_i : 0 + s = m[3] ? m[3].to_i : 0 return h * 3600 + mn * 60 + s end end @@ -500,17 +491,9 @@ def fetch_show_episodes(slug, name, feed_url) filename = safe_filename(title, guid[-20..]) file_path = download_episode(audio_url, dest_dir, filename) next if file_path.nil? - played_db[:episodes].insert_or_ignore( - show_slug: slug, guid: guid, title: title, - file_path: file_path, enclosure_url: audio_url, - runlength: duration, played: 0 - ) + insert_episode(played_db, slug, guid, title, file_path, audio_url, duration) else - played_db[:episodes].insert_or_ignore( - show_slug: slug, guid: guid, title: title, - file_path: nil, enclosure_url: audio_url, - runlength: duration, played: 0 - ) + insert_episode(played_db, slug, guid, title, nil, audio_url, duration) end new_count += 1 kind = archived ? "downloaded" : "live" @@ -522,6 +505,19 @@ def fetch_show_episodes(slug, name, feed_url) new_count end +def insert_episode(db, slug, guid, title, file_path, enclosure_url, duration) + # INSERT OR IGNORE semantics via the UNIQUE(show_slug, guid) constraint. + db.transaction do + db[:episodes].insert( + show_slug: slug, guid: guid, title: title, + file_path: file_path, enclosure_url: enclosure_url, + runlength: duration, played: 0 + ) + end +rescue Sequel::UniqueConstraintViolation + # Already recorded; ignore. +end + def fetch_all_episodes db = connect_subs shows = db[:shows].order(:name).all @@ -575,10 +571,14 @@ def add_show(feed_url) slug = slugify(name) guid = gen_uuid db = connect_subs - db[:shows].insert_or_ignore( - slug: slug, guid: guid, name: name, feed_url: feed_url, - source: "manual", opml_import: 0, archived: 1, media_class: cls - ) + begin + db[:shows].insert( + slug: slug, guid: guid, name: name, feed_url: feed_url, + source: "manual", opml_import: 0, archived: 1, media_class: cls + ) + rescue Sequel::UniqueConstraintViolation + # already present + end db.disconnect $log.info("Added show: #{name} (#{slug}) [#{cls}]") fetch_show_episodes(slug, name, feed_url) @@ -668,10 +668,6 @@ def run_fetch(config) fetch_all_episodes end -require "fileutils" -require "cgi" -require "base64" - def main args = ARGV.dup option = args.shift @@ -698,15 +694,14 @@ def main when "--import-opml" import_opml(args.first) else - needs_lock = true - if needs_lock && !acquire_lock! + if !acquire_lock! $log.info("Another radio process holds the lock; skipping this run.") return end begin run_fetch(config) ensure - release_lock! if needs_lock + release_lock! end end end diff --git a/install_for_jruby b/install_for_jruby index 2a031ec..655e4c1 100755 --- a/install_for_jruby +++ b/install_for_jruby @@ -104,7 +104,6 @@ echo "==> Installing gems one at a time into ${GEMS_DIR} ..." "$JRUBY_BIN" -S gem install --no-document sequel "$JRUBY_BIN" -S gem install --no-document json -# Give the service user ownership of the gem cache so cron runs work. chown -R "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "$GEMS_DIR" # --------------------------------------------------------------------------- @@ -207,57 +206,65 @@ chown "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "$CONFIG_JSON" # --------------------------------------------------------------------------- # 9. Initialize BOTH SQLite databases with the full current schema. -# Uses Sequel over jdbc-sqlite3, matching how the Ruby scripts access the -# DB at runtime. Idempotent via IF NOT EXISTS semantics. +# Uses raw CREATE TABLE IF NOT EXISTS via Database#execute - avoids the +# Sequel create_table DSL, which is unreliable under JRuby (instance_exec'd +# generator methods can resolve to nil). Works on CRuby and JDBC alike. # --------------------------------------------------------------------------- echo "==> Initializing SQLite databases via Sequel/jdbc-sqlite3 ..." GEM_HOME="$GEMS_DIR" GEM_PATH="$GEMS_DIR" "$JRUBY_BIN" -e ' require "sequel" -state_dir = ARGV[0] -subs_path = File.join(state_dir, "subscriptions.db") +state_dir = ARGV[0] +subs_path = File.join(state_dir, "subscriptions.db") played_path = File.join(state_dir, "played.db") -db = Sequel.connect("jdbc:sqlite:#{subs_path}") -db.extension :pragma -db.pragma journal_mode: :wal -db.pragma busy_timeout: 5000 -unless db.table_exists?(:shows) - db.create_table(:shows) do |t| - t.primary_key :slug, type: :string - t.string :guid, null: false, unique: true - t.string :name, null: false - t.string :feed_url, null: false, unique: true - t.string :source, default: "manual" - t.integer :opml_import, default: 0 - t.integer :archived, default: 1 - t.string :media_class - t.string :created_at - end +def tune(db) + db.execute("PRAGMA journal_mode=WAL;") + db.execute("PRAGMA busy_timeout=5000;") end + +SHOWS_SQL = <<~SQL + CREATE TABLE IF NOT EXISTS shows ( + slug TEXT PRIMARY KEY, + guid TEXT NOT NULL UNIQUE, + name TEXT NOT NULL, + feed_url TEXT NOT NULL UNIQUE, + source TEXT DEFAULT '"'"'manual'"'"', + opml_import INTEGER DEFAULT 0, + archived INTEGER DEFAULT 1, + media_class TEXT, + created_at TEXT + ); +SQL + +EPISODES_SQL = <<~SQL + CREATE TABLE IF NOT EXISTS episodes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + show_slug TEXT NOT NULL, + guid TEXT NOT NULL, + title TEXT, + file_path TEXT, + enclosure_url TEXT, + runlength INTEGER, + played INTEGER DEFAULT 0, + played_at TEXT, + UNIQUE (show_slug, guid) + ); +SQL + +INDEX_SQL = <<~SQL + CREATE INDEX IF NOT EXISTS idx_episodes_show_played ON episodes (show_slug, played); +SQL + +db = Sequel.connect("jdbc:sqlite:#{subs_path}") +tune(db) +db.execute(SHOWS_SQL) db.disconnect db = Sequel.connect("jdbc:sqlite:#{played_path}") -db.extension :pragma -db.pragma journal_mode: :wal -db.pragma busy_timeout: 5000 -unless db.table_exists?(:episodes) - db.create_table(:episodes) do |t| - t.primary_key :id - t.string :show_slug, null: false - t.string :guid, null: false - t.string :title - t.string :file_path - t.string :enclosure_url - t.integer :runlength - t.integer :played, default: 0 - t.string :played_at - t.unique_constraint %i[show_slug guid] - end -end -unless db.index_exists?(:episodes, [:show_slug, :played]) - db.create_index(:episodes, [:show_slug, :played], name: :idx_episodes_show_played) -end +tune(db) +db.execute(EPISODES_SQL) +db.execute(INDEX_SQL) db.disconnect puts " subscriptions.db and played.db initialized." @@ -296,13 +303,12 @@ exec("liquidsoap", File.expand_path("station.liq")) RBRUN chown "$LIQUIDSOAP_USER":"$LIQUIDSOAP_USER" "${INSTALL_DIR}/station.rb" -# Allow the service user to traverse the install dir. chmod o+x "$INSTALL_DIR" systemctl daemon-reload # --------------------------------------------------------------------------- -# 11. Cron jobs — installed into the liquidsoap user's crontab so all +# 11. Cron jobs - installed into the liquidsoap user's crontab so all # database/media writes happen under the same identity as the service. # --------------------------------------------------------------------------- echo "==> Setting up cron jobs under user '${LIQUIDSOAP_USER}' ..." @@ -311,14 +317,12 @@ UPDATE_PREFIX="cd ${INSTALL_DIR} && GEM_HOME=${GEMS_DIR} GEM_PATH=${GEMS_DIR} ${ FETCH_CRON="0 * * * * ${FETCH_PREFIX} >> ${STORAGE_PATH}/logs/fetch.log 2>&1" UPDATE_CRON="30 * * * * ${UPDATE_PREFIX} >> ${STORAGE_PATH}/logs/update.log 2>&1" -# Remove any stale entries from the liquidsoap user's crontab, then add ours. sudo -u "$LIQUIDSOAP_USER" crontab -l 2>/dev/null | grep -vF "fetch_podcasts.rb" | grep -vF "update_playlists.rb" > /tmp/cron_ls_rb.$$ || true echo "$FETCH_CRON" >> /tmp/cron_ls_rb.$$ echo "$UPDATE_CRON" >> /tmp/cron_ls_rb.$$ sudo -u "$LIQUIDSOAP_USER" crontab /tmp/cron_ls_rb.$$ rm -f /tmp/cron_ls_rb.$$ -# Also scrub these from root's crontab in case an earlier install put them there. crontab -l 2>/dev/null | grep -vF "fetch_podcasts.rb" | grep -vF "update_playlists.rb" > /tmp/cron_root_rb.$$ || true crontab /tmp/cron_root_rb.$$ rm -f /tmp/cron_root_rb.$$ diff --git a/update_playlists.rb b/update_playlists.rb index f4889bb..1a29751 100755 --- a/update_playlists.rb +++ b/update_playlists.rb @@ -12,7 +12,7 @@ require "json" require "logger" require "time" -ROOT = File.expand_path(File.dirname(__file__)) +ROOT = File.expand_path(File.dirname(__FILE__)) CONFIG_PATH = File.join(ROOT, "config.json") $state_dir = nil @@ -40,7 +40,7 @@ def init_paths! end $log = Logger.new(STDOUT) -$log.formatter = proc { |msg, _sev, _time, _prog| "#{Time.now} [INFO] #{msg}\n" } +$log.formatter = proc { |msg, _severity, _time, _progname| "#{Time.now} [INFO] #{msg}\n" } def log_error(msg) $log.error(msg) @@ -51,6 +51,9 @@ def setup_logging! Logger::LogDevice.new([STDOUT, File.join($logs_dir, "update.log")])) end +# --------------------------------------------------------------------------- +# Schema: single source of truth, applied idempotently via Sequel. +# --------------------------------------------------------------------------- SHOWS_COLUMNS = { slug: { type: :string, primary_key: true }, guid: { type: :string, null: false, unique: true }, @@ -60,7 +63,7 @@ SHOWS_COLUMNS = { opml_import: { type: :integer, default: 0 }, archived: { type: :integer, default: 1 }, media_class: { type: :string }, - created_at: { type: :string, default: Sequel.function(:datetime, "'now'") } + created_at: { type: :string } }.freeze EPISODES_COLUMNS = { @@ -75,20 +78,24 @@ EPISODES_COLUMNS = { played_at: { type: :string } }.freeze +def tune(db) + # Plain-SQL pragmas via Database#execute, which works on CRuby and JRuby/JDBC + # across all modern Sequel versions (the :pragma extension is CRuby-only and + # db.sql requires Sequel >= 5.42). + db.execute("PRAGMA journal_mode=WAL;") + db.execute("PRAGMA busy_timeout=5000;") +end + def connect_subs db = Sequel.connect("jdbc:sqlite:#{$subs_db_path}") - db.extension :pragma - db.pragma journal_mode: :wal - db.pragma busy_timeout: 5000 + tune(db) ensure_schema!(db, :shows, SHOWS_COLUMNS) db end def connect_played db = Sequel.connect("jdbc:sqlite:#{$played_db_path}") - db.extension :pragma - db.pragma journal_mode: :wal - db.pragma busy_timeout: 5000 + tune(db) ensure_schema!(db, :episodes, EPISODES_COLUMNS) unless db.index_exists?(:episodes, [:show_slug, :played]) db.create_index :episodes, [:show_slug, :played], name: :idx_episodes_show_played @@ -99,7 +106,9 @@ end def ensure_schema!(db, table, columns) unless db.table_exists?(table) db.create_table(table) do |t| - columns.each { |col, opts| t.column(col, **opts) } + columns.each do |col, opts| + t.column(col, **opts) + end t.unique_constraint %i[show_slug guid] if table == :episodes end return @@ -111,6 +120,9 @@ def ensure_schema!(db, table, columns) end end +# --------------------------------------------------------------------------- +# Mutual exclusion via flock on a shared lockfile. +# --------------------------------------------------------------------------- $lock_fh = nil def acquire_lock! @@ -139,6 +151,9 @@ def release_lock! end end +# --------------------------------------------------------------------------- +# Queue-file generation +# --------------------------------------------------------------------------- def annotate_uri(runlength, title, uri) def esc(v) '"' + v.to_s.gsub('"', '\\"') + '"' @@ -157,7 +172,7 @@ end def mark_as_played(slug, guid, played_db) played_db[:episodes] .where(show_slug: slug, guid: guid) - .update(played: 1, played_at: Sequel.function(:datetime, "'now'")) + .update(played: 1, played_at: Time.now.utc.strftime("%Y-%m-%d %H:%M:%S")) end def write_queue_file(slug, ep, archived) @@ -200,8 +215,7 @@ def json_summary result = {} shows.each do |show| slug = show[:slug] - counts = played_db[:episodes].where(show_slug: slug).hash_and_count - total = counts.values.sum + total = played_db[:episodes].where(show_slug: slug).count unplayed = played_db[:episodes].where(show_slug: slug, played: 0).count result[slug] = { name: show[:name], @@ -241,4 +255,3 @@ def main end main if __FILE__ == $PROGRAM_NAME -