diff --git a/fetch_podcasts.rb b/fetch_podcasts.rb index 3936efd..398dddc 100755 --- a/fetch_podcasts.rb +++ b/fetch_podcasts.rb @@ -1,634 +1,497 @@ #!/usr/bin/env jruby # frozen_string_literal: true -# -# fetch_podcasts.rb - Podcast subscription management for the radio automation. -# -# Manages the shows table in subscriptions.db and the episodes table in played.db. -# Automatically syncs with gpodder.net on every invocation (if enabled in config). -# -# Usage: -# fetch_podcasts.rb --add-show -# fetch_podcasts.rb --list [--detail] -# fetch_podcasts.rb --remove -# fetch_podcasts.rb --fetch-all -# fetch_podcasts.rb --import-opml -# fetch_podcasts.rb --archive -# fetch_podcasts.rb --unarchive -require "json" require "net/http" require "uri" -require "digest/md5" +require "json" +require "digest/sha1" require "sequel" require "nokogiri" -SCRIPT_DIR = File.expand_path(File.dirname(__FILE__)) +SCRIPT_DIR = File.expand_path(File.dirname(__FILE__)) CONFIG_PATH = File.join(SCRIPT_DIR, "config.json") def load_config - raw = JSON.parse(File.read(CONFIG_PATH)) - storage = raw["storage"] || "/mnt/storage/radio" - icecast = raw["icecast"] || {} - gpodder = raw["gpodder"] || {} - { - :storage => storage, - :icecast_host => icecast["host"] || "127.0.0.1", - :icecast_port => icecast["port"].to_i, - :icecast_mount => icecast["mount"] || "/data", - :icecast_source_user => icecast["source_username"] || "source", - :icecast_source_pass => icecast["source_password"] || "", - :gpodder_enable => gpodder["enable"] == true, - :gpodder_host => gpodder["host"] || "https://gpodder.net", - :gpodder_user => gpodder["username"] || "", - :gpodder_pass => gpodder["password"] || "", - :gpodder_device_id => gpodder["device_id"] || "" - } + JSON.parse(File.read(CONFIG_PATH)) end -CFG = load_config -STORAGE_ROOT = CFG[:storage] -STATE_DIR = File.join(STORAGE_ROOT, "state") -LOGS_DIR = File.join(STORAGE_ROOT, "logs") -TMP_DIR = File.join(STATE_DIR, "tmp") -SUBS_DB = File.join(STATE_DIR, "subscriptions.db") -PLAYED_DB = File.join(STATE_DIR, "played.db") -LOG_FILE = File.join(LOGS_DIR, "fetch_cron.log") +CFG = load_config +STORAGE = CFG["storage"] +STATE_DIR = File.join(STORAGE, "state") +LOGS_DIR = File.join(STORAGE, "logs") +PODCAST_DIR = File.join(STORAGE, "podcasts") +SUBS_DB = File.join(STATE_DIR, "subscriptions.db") +PLAYED_DB = File.join(STATE_DIR, "played.db") +LOG_FILE = File.join(LOGS_DIR, "fetch_podcasts.log") + +$global_log_dir = LOGS_DIR def ensure_dir(path) return if Dir.exist?(path) parent = File.dirname(path) - unless path.start_with?("/") - raise ArgumentError, "ensure_dir requires an absolute path, got: #{path}" - end - components = path.split("/").reject(&:empty?) - current = "/" - components.each do |comp| - current = File.join(current, comp) - unless Dir.exist?(current) - begin - Dir.mkdir(current) - rescue Errno::EEXIST - nil - end - end - end + ensure_dir(parent) unless Dir.exist?(parent) && parent != path + Dir.mkdir(path) +rescue Errno::EEXIST + nil end -[STORAGE_ROOT, STATE_DIR, LOGS_DIR, TMP_DIR].each { |d| ensure_dir(d) } +ensure_dir(STATE_DIR) +ensure_dir(LOGS_DIR) +ensure_dir(PODCAST_DIR) -$log_fh = File.open(LOG_FILE, "a+") +$db_s = Sequel.connect("jdbc:sqlite:" + SUBS_DB) +$db_p = Sequel.connect("jdbc:sqlite:" + PLAYED_DB) + +$db_s.execute("PRAGMA journal_mode=WAL;") +$db_s.execute("PRAGMA busy_timeout=5000;") +$db_p.execute("PRAGMA journal_mode=WAL;") +$db_p.execute("PRAGMA busy_timeout=5000;") def log(level, msg) ts = Time.now.strftime("%Y-%m-%d %H:%M:%S") line = "[#{ts}] [#{level}] #{msg}" - $log_fh.write(line + "\n") - $log_fh.flush puts line -rescue Exception => e - puts "LOG ERROR: #{e.class} #{e.message}" -end - -$db_s = Sequel.connect("jdbc:sqlite:" + SUBS_DB) -$db_p = Sequel.connect("jdbc:sqlite:" + PLAYED_DB) -def table_count(db, tbl_name, col = :name) - db[:sqlite_master].where(type: "table", name: tbl_name).count -rescue Exception => e - log("WARN", "count check failed for #{tbl_name}: #{e.class} #{e.message}") - 0 -end - -def ensure_schema begin - if table_count($db_s, "shows").zero? - $db_s.execute <<-SQL - CREATE TABLE shows ( - guid TEXT PRIMARY KEY, - slug TEXT UNIQUE NOT NULL, - title TEXT NOT NULL, - feed_url TEXT NOT NULL, - audio_only INTEGER DEFAULT 1, - archive INTEGER DEFAULT 0, - opml_import INTEGER DEFAULT 0, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - SQL - log("INFO", "Created 'shows' table") - end - - if table_count($db_p, "episodes").zero? - $db_p.execute <<-SQL - CREATE TABLE episodes ( - guid TEXT PRIMARY KEY, - show_guid TEXT NOT NULL, - title TEXT, - enclosure_url TEXT, - duration_seconds INTEGER, - published_date TEXT, - played INTEGER DEFAULT 0, - downloaded INTEGER DEFAULT 0, - local_path TEXT - ) - SQL - log("INFO", "Created 'episodes' table") - end - rescue Exception => e - log("ERROR", "Schema setup failed: #{e.class} #{e.message}") - raise e + File.open(LOG_FILE, "a") { |f| f.puts(line) } + rescue StandardError => e + warn "Log write failed: #{e.class} #{e.message}" end end -def table_exists?(db, tbl_name) - begin - db.from(:sqlite_master).where(name: tbl_name).count > 0 - rescue Exception => e - log("WARN", "table_exists? check failed for '#{tbl_name}': #{e.class} #{e.message}") - false +def table_count(db, tbl) + db[:sqlite_master].where(type: 'table', name: tbl).count > 0 +end + +def ensure_schema + unless table_count($db_s, "shows") + $db_s.execute(%( + CREATE TABLE IF NOT EXISTS shows ( + guid TEXT PRIMARY KEY, + slug TEXT UNIQUE NOT NULL, + title TEXT NOT NULL, + feed_url TEXT NOT NULL, + archive INTEGER DEFAULT 0, + opml_import INTEGER DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + )) + log("INFO", "Created shows table in subscriptions.db") + end + + unless table_count($db_p, "episodes") + $db_p.execute(%( + CREATE TABLE IF NOT EXISTS episodes ( + guid TEXT PRIMARY KEY, + show_guid TEXT NOT NULL, + title TEXT NOT NULL, + url TEXT NOT NULL, + duration_seconds INTEGER DEFAULT 0, + played INTEGER DEFAULT 0, + local_path TEXT, + file_size_bytes INTEGER DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + )) + log("INFO", "Created episodes table in played.db") end end ensure_schema -def make_slug(title_str, feed_url) - base = title_str.to_s.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/^_+|_+$/, "") - base = "show" if base.empty? - suffix = Digest::MD5.hexdigest(feed_url)[0..3] - candidate = "#{base}_#{suffix}" - existing = $db_s[:shows].select_map(:slug) - i = 1 - while existing.include?(candidate) - i += 1 - candidate = "#{base}_#{suffix}_#{i}" - end - candidate +def gen_guid(seed_str) + Digest::SHA1.hexdigest(seed_str.to_s) end -def gen_guid(seed_str) - Digest::MD5.hexdigest(seed_str) +def generate_slug(title) + title.to_s.downcase.gsub(/[^a-z0-9]+/, "_").gsub(/^_+|_+$/, "") end def http_stream_to_file(url, dest_path, user = nil, pass = nil) uri = URI.parse(url) - req = Net::HTTP::Get.new(uri.path + (uri.query ? "?#{uri.query}" : "")) - if user && pass - req.basic_auth(user, pass) - end - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = (uri.scheme == "https") - http.open_timeout = 30 - http.read_timeout = 60 success = false + tmp_dest = "#{dest_path}.downloading" + begin - http.request(req) do |r| - if r.is_a?(Net::HTTPSuccess) - File.open(dest_path, "wb") do |f| - r.read_body { |chunk| f.write(chunk) } - end - size = File.size(dest_path) - log("INFO", "Downloaded #{size} bytes from #{url}") - success = true - else - log("ERROR", "HTTP #{r.code} fetching #{url}") + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = (uri.scheme == "https") + http.open_timeout = 30 + http.read_timeout = 60 + + req = Net::HTTP::Get.new(uri.request_uri) + req.add_field("User-Agent", "RadioAutomation/1.0 (JRuby)") + req.basic_auth(user, pass) if user && !user.empty? + + res = nil + http.start do |conn| + conn.request(req) do |response| + res = response end end - rescue Exception => e - log("ERROR", "Failed to download #{url}: #{e.class} #{e.message}") + + if res.is_a?(Net::HTTPRedirection) + loc = res["location"] + log("WARN", "Redirect to: #{loc}") + return http_stream_to_file(loc, dest_path, user, pass) + end + + unless res.is_a?(Net::HTTPSuccess) + raise "HTTP #{res.code}: #{res.message}" + end + + File.open(tmp_dest, "wb") do |f| + res.read_body { |chunk| f.write(chunk) } + end + File.rename(tmp_dest, dest_path) + success = true + rescue Exception => error + log("ERROR", "Download failed for #{url}: #{error.class}: #{error.message}") + begin + File.delete(tmp_dest) if File.exist?(tmp_dest) + rescue StandardError + # ignore cleanup errors + end end success end -def parse_feed(data={}) - result = {:title => "Untitled Show", :description => "", :episodes => []} - unless data.is_a?(::Hash) - raise "You MUST provide a Hash with file_path and feed_url specified. You passed #{data.inspect}" - end - file_path = data[:file_path] - unless file_path - raise "You MUST provide a Hash with file_path and feed_url specified. You passed #{data.inspect}" - end - feed_url = data[:feed_url] - unless feed_url - raise "You MUST provide a Hash with file_path and feed_url specified. You passed #{data.inspect}" - end - # - document = Nokogiri::XML(File.read(file_path)) - unless document - raise "Failed to import XML document: #{file_path.inspect}" - end - channel = document.at_xpath("//channel") - if channel.nil? - log("ERROR", "No found in feed at #{feed_url}") - else - title_element = channel.at_xpath("./title") - description_element = channel.at_xpath("./description") - show_title = title_element ? title_element.text.strip : "Untitled Show" - show_description = description_element ? description_element.text.strip : "" - # - episodes = [] - channel.elements("item").each do |item| - # - episode_title_element = item.at_xpath("./title") - enclosure_element = item.at_xpath("./enclosure") - guid_element = item.at_xpath("./guid") - duration_element = item.at_xpath(".//duration") - publication_date_element = item.at_xpath("./pubDate") - # - episode_title = episode_title_element ? episode_title_element.text.strip : "Untitled" - enclosure_url = enclosure_element ? enclosure_element.attribute("url").to_s.strip : "" - enclosure_type = enclosure_element ? enclosure_element.attribute("type").to_s.strip : "" - enclosure_length = enclosure_element ? enclosure_element.attribute("length").to_s.strip.to_i : 0 - episode_guid = guid_element ? guid_element.text.strip : "" - episode_duration = duration_element ? duration_element.text.strip.to_i : 0 - episode_publication_date = publication_date_element ? publication_date_element.text.strip : "" - # - if enclosure_url.empty? || enclosure_type =~ /video/i - next - else - # - if episode_guid.empty? - episode_guid = gen_guid("#{enclosure_url}#{episode_title}") - end - if episode_duration <= 0 && enclosure_length > 0 - episode_duration = (enclosure_length / (128 * 1024)).to_i - end - # - episodes << {:guid => episode_guid, :title => episode_title, :url => enclosure_url, :duration_seconds => episode_duration, :published_at => episode_publication_date, :file_size_bytes => enclosure_length} - # - end - # - end - # - result = {:title => show_title, :description => show_description, :episodes => episodes} - # - end - # - result -end +def parse_feed(opts) + file_path = opts[:file_path] || raise("parse_feed: :file_path is required") + feed_url = opts[:feed_url] || "" - -def parse_feed_from_file(file_path="", feed_url="") - doc = Nokogiri::XML(File.read(file_path)) + doc = Nokogiri::XML(File.read(file_path), nil, XML::NO_NETWORK) channel = doc.at_xpath("//channel") - if channel.nil? - log("ERROR", "No found in feed at #{feed_url}") - return nil - end + raise "No element found in feed" unless channel - title_el = channel.at_xpath("./title") - desc_el = channel.at_xpath("./description") - show_title = title_el ? title_el.text.strip : "Unknown Show" - show_desc = desc_el ? desc_el.text.strip : "" - - episodes = [] + items = [] channel.elements("item").each do |item| - ep_title_el = item.at_xpath("./title") - enc_el = item.at_xpath("./enclosure") - guid_el = item.at_xpath("./guid") - dur_el = item.at_xpath(".//duration") - pub_el = item.at_xpath("./pubDate") + enc = item.at_xpath("./enclosure") + next unless enc + next unless enc.attributes["type"].value =~ /audio/i - ep_title = ep_title_el ? ep_title_el.text.strip : "Untitled" - enc_url = enc_el ? enc_el.attribute("url").to_s.strip : "" - enc_type = enc_el ? enc_el.attribute("type").to_s.strip : "" - enc_len = enc_el ? enc_el.attribute("length").to_s.strip.to_i : 0 - ep_guid = guid_el ? guid_el.text.strip : "" - ep_dur = dur_el ? dur_el.text.strip.to_i : 0 - ep_pub = pub_el ? pub_el.text.strip : "" - - next if enc_url.empty? - - if enc_type =~ /video/i - next + pub_date_raw = item.at_xpath("./pubDate")&.text + published_at = begin + Time.parse(pub_date_raw)&.utc&.strftime("%Y-%m-%d %H:%M:%S") + rescue StandardError + nil end - if ep_guid.empty? - ep_guid = gen_guid("#{enc_url}#{ep_title}") + dur_el = item.at_xpath(".//media:duration", "media" => "http://search.yahoo.com/mrss/") + dur_sec = 0 + if dur_el + d = dur_el.text.strip + if d.include?(":") + parts = d.split(":").map(&:to_i) + dur_sec = parts[0] * 3600 + parts[1] * 60 + parts[2] + else + dur_sec = d.to_i + end end - if ep_dur <= 0 && enc_len > 0 - ep_dur = (enc_len / (128 * 1024)).to_i - end + enc_len = enc.attributes["length"]&.value&.to_i || 0 + est_dur = (enc_len / (128 * 1024)).round if enc_len > 0 - episodes << { - :guid => ep_guid, - :title => ep_title, - :url => enc_url, - :duration_seconds => ep_dur, - :published_at => ep_pub, - :file_size_bytes => enc_len + items << { + guid: item.at_xpath("./guid")&.text.presence || gen_guid(enc.attributes["url"].value), + title: item.at_xpath("./title")&.text || "Untitled", + url: enc.attributes["url"].value, + duration_seconds: dur_sec > 0 ? dur_sec : (est_dur || 0), + file_size_bytes: enc_len, + published_at: published_at } end { - :title => show_title, - :description => show_desc, - :episodes => episodes + title: channel.at_xpath("./title")&.text || "Unknown Show", + feed_url: feed_url, + items: items } end -def register_show(parsed, feed_url) - slug = make_slug(parsed[:title], feed_url) - show_guid = gen_guid(feed_url) - existing = $db_s[:shows].where(slug: slug).first +def register_show(show_title, feed_url, archive_flag = 0, opml_flag = 0) + url = feed_url.to_s.strip + return false if url.empty? + + existing = $db_s[:shows].where(feed_url: url).first if existing - log("INFO", "Show already registered: #{parsed[:title]} (#{slug})") - return existing[:guid] + # Update archive/opml flags if they changed + if existing[:archive] != archive_flag || existing[:opml_import] != opml_flag + $db_s[:shows].where(guid: existing[:guid]).update( + archive: archive_flag, + opml_import: opml_flag + ) + log("DEBUG", "Updated flags for #{existing[:slug]}: archive=#{archive_flag}, opml=#{opml_flag}") + end + return existing end - $db_s[:shows].insert(guid: show_guid, slug: slug, title: parsed[:title], feed_url: feed_url, audio_only: 1, archive: 0) - log("INFO", "Registered show: #{parsed[:title]} (#{slug})") + title = show_title.to_s.strip + slug = generate_slug(title) + slug = "show_#{Digest::SHA1.hexdigest(url)[0,8]}" if slug.empty? - parsed[:episodes].each do |ep| - $db_p[:episodes].insert( - guid: ep[:guid], - show_guid: show_guid, - title: ep[:title], - url: ep[:url], - duration_seconds: ep[:duration_seconds], - published_at: ep[:published_at], - played: 0, - downloaded: 0, - file_size_bytes: ep[:file_size_bytes] - ) - end + guid = gen_guid(url) + now = Time.now.utc.strftime("%Y-%m-%d %H:%M:%S") - log("INFO", "Stored #{parsed[:episodes].size} episodes for #{slug}") - show_guid + $db_s[:shows].insert( + guid: guid, + slug: slug, + title: title, + feed_url: url, + archive: archive_flag, + opml_import: opml_flag, + created_at: now + ) + log("INFO", "Registered show: #{slug} (#{title})") + $db_s[:shows].where(guid: guid).first end -def cmd_add_show(url) - log("INFO", "Adding show: #{url}") - tmp_file = File.join(TMP_DIR, "feed_#{Process.pid}.xml") - ok = http_stream_to_file(url, tmp_file) +def sync_gpodder + gpodder_cfg = CFG["gpodder"] || {} + return unless gpodder_cfg["enabled"] == true + + host = gpodder_cfg["host"].to_s.sub(/\Ahttps?:\/\//, "").sub(/\/.*\z/, "") + username = gpodder_cfg["username"].to_s + password = gpodder_cfg["password"].to_s + device_id = gpodder_cfg["device_id"].to_s + return if username.empty? || password.empty? || device_id.empty? + + full_host = "https://#{host}" + opml_url = "#{full_host}/subscriptions/#{username}/#{device_id}.opml" + log("INFO", "gPodder sync: fetching OPML from #{opml_url}") + + tmp_opml = "/tmp/radio_gp_odder_sync.opml" + ok = http_stream_to_file(opml_url, tmp_opml, username, password) unless ok - log("ERROR", "Could not download feed: #{url}") - exit 1 - end - # ??? - parsed = parse_feed({:file_path => tmp_file, :feed_url => url}) - # parsed = parse_feed_from_file(tmp_file, url) - begin - File.delete(tmp_file) - rescue Errno::ENOENT - nil - end - - if parsed.nil? - log("ERROR", "Could not parse feed: #{url}") - exit 1 - end - - register_show(parsed, url) - log("INFO", "Done adding show: #{parsed[:title]}") -end - -def cmd_list(detail) - rows = $db_s[:shows].all - if rows.empty? - puts "No shows registered." + log("ERROR", "gPodder sync: failed to download #{opml_url}") return end - rows.each do |r| - line = "%-30s %-40s arch=%d" % [r[:slug], r[:title], r[:archive]] - if detail - ep_count = $db_p[:episodes].where(show_guid: r[:guid]).count - played_count = $db_p[:episodes].where(show_guid: r[:guid], played: 1).count - line += " eps=#{ep_count} played=#{played_count}" + + opml_doc = Nokogiri::XML(File.read(tmp_opml), nil, XML::NO_NETWORK) + remote_shows = [] + opml_doc.xpath("//outline[@xmlUrl]").each do |o| + xml_url = o.attr("xmlUrl").to_s.strip + title = o.attr("title").to_s.strip + next if xml_url.empty? + next unless xml_url =~ /\.(rss|atom)(\?.*)?\z/i + remote_shows << { url: xml_url, title: title } + end + + added = 0 + removed = 0 + + remote_shows.each do |rs| + existing = $db_s[:shows].where(feed_url: rs[:url]).first + if existing.nil? + row = register_show(rs[:title], rs[:url], 0, 0) + added += 1 if row end - puts line + end + + all_local = $db_s[:shows].all + all_local.each do |row| + if row[:opml_import] == 0 + still_remote = remote_shows.any? { |rs| rs[:url] == row[:feed_url] } + unless still_remote + $db_p[:episodes].where(show_guid: row[:guid]).delete + $db_s[:shows].where(guid: row[:guid]).delete + log("INFO", "gPodder sync: pruned '#{row[:slug]}'") + removed += 1 + end + end + end + + log("INFO", "gPodder sync: #{added} added, #{removed} removed") + begin + File.delete(tmp_opml) + rescue StandardError + # ignore cleanup errors end end -def cmd_remove(slug_or_url) - row = $db_s[:shows].where(Sequel.or({slug: slug_or_url}, {feed_url: slug_or_url})).first +def cmd_add_show(args) + if args.size < 1 + log("ERROR", "Usage: fetch_podcasts.rb --add-show ") + exit 1 + end + url = args.first + title = url + parsed = nil + + tmp_feed = "/tmp/radio_feed_probe.xml" + if http_stream_to_file(url, tmp_feed) + parsed = parse_feed({:file_path => tmp_feed, :feed_url => url}) + title = parsed[:title] if parsed && !parsed[:title].empty? + begin + File.delete(tmp_feed) + rescue StandardError + # ignore cleanup errors + end + end + + row = register_show(title, url) + if row + log("INFO", "Added show: #{row[:slug]} (#{row[:title]})") + else + log("ERROR", "Failed to add show: #{url}") + exit 1 + end +end + +def cmd_remove_show(slug_or_url) + row = $db_s[:shows].where(Sequel.function("lower", Sequel.val(:slug)) =~ "%#{slug_or_url}%" ).first + row ||= $db_s[:shows].where(feed_url: slug_or_url).first if row.nil? log("ERROR", "Show not found: #{slug_or_url}") exit 1 end $db_p[:episodes].where(show_guid: row[:guid]).delete $db_s[:shows].where(guid: row[:guid]).delete - log("INFO", "Removed show #{row[:slug]} and its episodes") + log("INFO", "Removed show: #{row[:slug]} and its episodes") +end + +def cmd_archive(slug_or_url, flag) + row = $db_s[:shows].where(slug: slug_or_url).first + row ||= $db_s[:shows].where(feed_url: slug_or_url).first + if row.nil? + log("ERROR", "Show not found: #{slug_or_url}") + exit 1 + end + val = flag ? 1 : 0 + $db_s[:shows].where(guid: row[:guid]).update(archive: val) + log("INFO", "#{flag ? 'Archived' : 'Unarchived'}: #{row[:slug]}") +end + +def cmd_import_opml(path) + opml_file = path + raise "File not found: #{opml_file}" unless File.exist?(opml_file) + + doc = Nokogiri::XML(File.read(opml_file)) { |config| config.nonet } + + outlines = doc.xpath("//outline") + count = 0 + outlines.each do |ol| + url = ol.attr("xmlUrl") || ol.attr("url") + title = ol.attr("title") || "Unknown" + next if url.nil? || url.empty? + + existing = $db_s[:shows].where(feed_url: url).first + if existing + log("info", "OPML: '#{existing[:slug]}' already registered, skipping") + next + end + + slug = gen_guid(url)[0..11] + guid = gen_guid(url) + $db_s[:shows].insert( + guid: guid, + slug: slug, + title: title, + feed_url: url, + archive: 1, + opml_import: 1, + created_at: Time.now.utc.strftime("%Y-%m-%d %H:%M:%S") + ) + count += 1 + log("info", "OPML: registered '#{slug}' (#{title})") + end + + log("info", "Imported #{count} shows from OPML") +end + +def cmd_list(detail) + rows = $db_s[:shows].order(:slug).all + if rows.empty? + puts "No shows registered." + return + end + if detail + rows.each do |r| + ep_count = $db_p[:episodes].where(show_guid: r[:guid]).count + unplayed = $db_p[:episodes].where(show_guid: r[:guid], played: 0).count + printf("%-40s %-60s arch=%d eps=%d unplayed=%d\n", + r[:slug][0,40], r[:title][0,60], r[:archive], ep_count, unplayed) + end + else + rows.each do |r| + printf("%-40s %-60s arch=%d\n", r[:slug][0,40], r[:title][0,60], r[:archive]) + end + end end def cmd_fetch_all - rows = $db_s[:shows].all - if rows.empty? - log("INFO", "No shows to fetch.") + shows = $db_s[:shows].where(archive: 1).all + if shows.empty? + log("INFO", "No archived shows to fetch.") return end - rows.each do |show| - log("INFO", "Fetching: #{show[:title]} (#{show[:slug]})") - tmp_file = File.join(TMP_DIR, "feed_#{show[:slug]}_#{Process.pid}.xml") - ok = http_stream_to_file(show[:feed_url], tmp_file) - unless ok - log("ERROR", "Could not download feed: #{show[:feed_url]}") + shows.each do |show| + log("INFO", "Fetching: #{show[:title]}") + tmp_feed = "/tmp/radio_fetch_#{show[:slug]}.xml" + unless http_stream_to_file(show[:feed_url], tmp_feed) + log("WARN", "Feed download failed for #{show[:slug]}, skipping") next end - # ??? - parsed = parse_feed({:file_path => tmp_file, :feed_url => show[:feed_url]}) - # parsed = parse_feed_from_file(tmp_file, show[:feed_url]) + + parsed = parse_feed({:file_path => tmp_feed, :feed_url => show[:feed_url]}) begin - File.delete(tmp_file) - rescue Errno::ENOENT - nil - end - if parsed.nil? - log("ERROR", "Could not parse feed: #{show[:feed_url]}") - next + File.delete(tmp_feed) + rescue StandardError + # ignore cleanup errors end + + next unless parsed + + show_dir = File.join(PODCAST_DIR, show[:slug]) + ensure_dir(show_dir) + new_eps = 0 - parsed[:episodes].each do |ep| + parsed[:items].each do |ep| existing = $db_p[:episodes].where(guid: ep[:guid]).first - if existing.nil? - $db_p[:episodes].insert( - guid: ep[:guid], - show_guid: show[:guid], - title: ep[:title], - url: ep[:url], - duration_seconds: ep[:duration_seconds], - published_at: ep[:published_at], - played: 0, - downloaded: 0, - file_size_bytes: ep[:file_size_bytes] - ) - new_eps += 1 + if existing + next end + + filename = "#{ep[:guid]}.mp3" + dest = File.join(show_dir, filename) + dl_ok = http_stream_to_file(ep[:url], dest) + size = dl_ok ? File.size(dest) : 0 + + now = Time.now.utc.strftime("%Y-%m-%d %H:%M:%S") + $db_p[:episodes].insert( + guid: ep[:guid], + show_guid: show[:guid], + title: ep[:title], + url: ep[:url], + duration_seconds: ep[:duration_seconds], + played: 0, + local_path: dest, + file_size_bytes: size, + created_at: now + ) + new_eps += 1 end - log("INFO", "#{new_eps} new episodes for #{show[:slug]}") + log("INFO", "#{show[:slug]}: #{new_eps} new episodes downloaded") end end -def cmd_import_opml(opml_file) - doc = Nokogiri::XML(File.read(opml_file)) - outlines = doc.xpath("//outline[@type='rss']") - count = 0 - outlines.each do |o| - url = o.attribute("xmlUrl").to_s.strip - next if url.empty? - cmd_add_show(url) - count += 1 - end - log("INFO", "Imported #{count} shows from OPML") +sync_gpodder + +command = ARGV.shift +case command +when "--list" + detail = ARGV.include?("--detail") + cmd_list(detail) +when "--add-show" + cmd_add_show(ARGV) +when "--remove" + cmd_remove_show(ARGV.first) +when "--archive" + cmd_archive(ARGV.first, true) +when "--unarchive" + cmd_archive(ARGV.first, false) +when "--import-opml" + cmd_import_opml(ARGV.first) +when "--fetch-all" + cmd_fetch_all +else + puts "Usage: fetch_podcasts.rb [--list|--detail|--add-show |--remove |--archive |--unarchive |--import-opml |--fetch-all]" end - -def cmd_archive(slug) - row = $db_s[:shows].where(slug: slug).first - if row.nil? - log("ERROR", "Show not found: #{slug}") - exit 1 - end - $db_s[:shows].where(guid: row[:guid]).update(archive: 1) - log("INFO", "Archived show: #{slug}") -end - -def cmd_unarchive(slug) - row = $db_s[:shows].where(slug: slug).first - if row.nil? - log("ERROR", "Show not found: #{slug}") - exit 1 - end - $db_s[:shows].where(guid: row[:guid]).update(archive: 0) - log("INFO", "Unarchived show: #{slug}") -end - -def sync_gpodder - unless CFG[:gpodder_enable] - return - end - - log("DEBUG", "Sync check: enable=#{CFG[:gpodder_enable]}, user=#{CFG[:gpodder_user]}, device=#{CFG[:gpodder_device_id]}") - - device_id = CFG[:gpodder_device_id].to_s.strip - if device_id.empty? - log("WARN", "gPodder sync enabled but no device_id configured; skipping.") - return - end - - host = CFG[:gpodder_host].to_s.strip - user = CFG[:gpodder_user].to_s.strip - pass = CFG[:gpodder_pass].to_s - full_host = host.start_with?("http") ? host : "https://#{host}" - api_url = "#{full_host}/subscriptions/#{user}/#{device_id}.opml" - tmp_file = File.join(TMP_DIR, "gpodder_sync_#{Process.pid}.opml") - - ok = http_stream_to_file(api_url, tmp_file, user, pass) - unless ok - log("WARN", "gPodder sync failed to download: #{api_url}") - return - end - - begin - doc = Nokogiri::XML(File.read(tmp_file)) - rescue Exception => e - log("WARN", "gPodder sync: failed to parse OPML: #{e.message}") - return - ensure - begin - File.delete(tmp_file) - rescue Errno::ENOENT - nil - end - end - - outlines = doc.xpath("//outline[@xmlUrl]").map do |o| - { - url: o.attr("xmlUrl").to_s.strip, - title: o.attr("title").to_s.strip - } - end - - remote_urls = outlines.map { |o| o[:url] }.select { |u| !u.empty? } - added = 0 - - outlines.each do |o| - next if o[:url].empty? - - existing = $db_s[:shows].where(feed_url: o[:url]).first - if existing.nil? - log("INFO", "gPodder sync: registering new show #{o[:title]}") - tmp_feed = File.join(TMP_DIR, "gpodder_feed_#{Process.pid}.xml") - fok = http_stream_to_file(o[:url], tmp_feed) - - if fok - # ??? - parsed = parse_feed({:file_path => tmp_feed, :feed_url => o[:url]}) - # parsed = parse_feed_from_file(tmp_feed, o[:url]) - - begin - File.delete(tmp_feed) - rescue Errno::ENOENT - nil - end - - if parsed - register_show(parsed, o[:url]) - added += 1 - end - end - end - end - - pruned = 0 - $db_s[:shows].all.each do |row| - if row[:opml_import].to_i == 0 && !remote_urls.include?(row[:feed_url].to_s) - $db_p[:episodes].where(show_guid: row[:guid]).delete - $db_s[:shows].where(guid: row[:guid]).delete - pruned += 1 - log("INFO", "gPodder sync: pruned '#{row[:slug]}'") - end - end - - log("INFO", "gPodder sync complete: #{added} added, #{pruned} removed") -end - -def main - args = ARGV.dup - command = args.shift - sync_gpodder - case command - when "--add-show" - url = args.shift - if url.nil? - puts "Usage: fetch_podcasts.rb --add-show " - exit 1 - end - cmd_add_show(url) - when "--list" - detail = args.include?("--detail") - cmd_list(detail) - when "--remove" - slug = args.shift - if slug.nil? - puts "Usage: fetch_podcasts.rb --remove " - exit 1 - end - cmd_remove(slug) - when "--fetch-all" - cmd_fetch_all - when "--import-opml" - opml_file = args.shift - if opml_file.nil? - puts "Usage: fetch_podcasts.rb --import-opml " - exit 1 - end - cmd_import_opml(opml_file) - when "--archive" - slug = args.shift - if slug.nil? - puts "Usage: fetch_podcasts.rb --archive " - exit 1 - end - cmd_archive(slug) - when "--unarchive" - slug = args.shift - if slug.nil? - puts "Usage: fetch_podcasts.rb --unarchive " - exit 1 - end - cmd_unarchive(slug) - else - puts "Usage: fetch_podcasts.rb [--add-show |--list|--remove |--fetch-all|--import-opml |--archive |--unarchive ]" - end -end - -main if __FILE__ == $PROGRAM_NAME - diff --git a/install_for_jruby b/install_for_jruby index 4267012..68af498 100755 --- a/install_for_jruby +++ b/install_for_jruby @@ -55,7 +55,35 @@ rm -f "${STORAGE_ROOT}/state/subscriptions.db" \ "${STORAGE_ROOT}/state/subscriptions.db-shm" \ "${STORAGE_ROOT}/state/played.db-wal" \ "${STORAGE_ROOT}/state/played.db-shm" -echo "==> Databases dropped (will be recreated by scripts on first run)" +echo "==> Databases dropped (will be recreated now)" +# Initialize database schemas +sqlite3 "${STORAGE_ROOT}/state/subscriptions.db" <<'SQL' +CREATE TABLE IF NOT EXISTS shows ( + guid TEXT PRIMARY KEY, + slug TEXT UNIQUE NOT NULL, + title TEXT, + feed_url TEXT UNIQUE NOT NULL, + archive INTEGER DEFAULT 0, + opml_import INTEGER DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +SQL + +sqlite3 "${STORAGE_ROOT}/state/played.db" <<'SQL' +CREATE TABLE IF NOT EXISTS episodes ( + guid TEXT PRIMARY KEY, + show_guid TEXT NOT NULL REFERENCES shows(guid), + title TEXT, + url TEXT, + duration_seconds INTEGER, + played INTEGER DEFAULT 0, + local_path TEXT, + file_size_bytes INTEGER DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +SQL + +echo "Database schemas initialized." # --- Install gems one at a time --------------------------------------------- export GEM_HOME="${GEMS_DIR}" @@ -155,6 +183,11 @@ WantedBy=multi-user.target EOF systemctl daemon-reload echo "==> Installed ${UNIT_PATH}" +# After the mkdir -p "$STORAGE/state" line, add: +mkdir -p "$STORAGE/state" +chown -R liquidsoap:liquidsoap "$STORAGE/state" +chmod 755 "$STORAGE/state" +chmod 664 "$STORAGE/state/"*.db 2>/dev/null || true # --- Cron entries (liquidsoap user's crontab already exists from package) ----- CRON_FETCH="0 * * * * cd ${INSTALL_DIR} && ${INSTALL_DIR}/run_radio.sh fetch_podcasts.rb --fetch-all >> ${STORAGE_ROOT}/logs/fetch_cron.log 2>&1" diff --git a/update_playlists.rb b/update_playlists.rb index adeb538..464c09c 100755 --- a/update_playlists.rb +++ b/update_playlists.rb @@ -1,21 +1,22 @@ #!/usr/bin/env jruby # frozen_string_literal: true # -# update_playlists.rb - Select the next unplayed episode per show and write -# annotated-URI queue files for station.liq to consume. +# update_playlists.rb - Select next unplayed episode per show and write queue files # -# Reads from played.db (episode records with played flag), writes queue files -# under /playlists/. Cycles archived shows when all episodes are -# already played. +# For each show in subscriptions.db: +# 1. Count total/unplayed episodes in played.db +# 2. If all played (and archive=0): reset all to played=0, pick first, mark played=1 +# 3. If unplayed > 0: pick one at random, mark played=1 +# 4. Write /queue/.txt with annotated URI or "SKIP" # # Usage: # ./update_playlists.rb [--json] -# -# Options: -# --json Emit a JSON summary of each show's episode counts to stdout. -require "json" require "sequel" +require "jdbc/sqlite3" +require "digest/sha1" +require "fileutils" +require "json" SCRIPT_DIR = File.expand_path(File.dirname(__FILE__)) CONFIG_PATH = File.join(SCRIPT_DIR, "config.json") @@ -23,196 +24,180 @@ CONFIG_PATH = File.join(SCRIPT_DIR, "config.json") def load_config raw = File.read(CONFIG_PATH) cfg = JSON.parse(raw) - raise "FATAL: storage missing from config.json" unless cfg["storage"] && !cfg["storage"].to_s.empty? - cfg + storage = cfg["storage"].to_s.strip + raise "Missing 'storage' in config.json" if storage.empty? + { + storage: storage, + icecast_port: cfg.dig("icecast", "port").to_i, + mount_point: cfg.dig("icecast", "mount_point").to_s, + source_user: cfg.dig("icecast", "source_username").to_s, + source_pass: cfg.dig("icecast", "source_password").to_s, + gpodder_host: cfg.dig("gpodder", "host").to_s, + gpodder_user: cfg.dig("gpodder", "username").to_s, + gpodder_pass: cfg.dig("gpodder", "password").to_s, + gpodder_device: cfg.dig("gpodder", "device_id").to_s, + gpodder_enable: cfg.dig("gpodder", "enable") == true + } end -CFG = load_config -STORAGE_ROOT = CFG["storage"] -STATE_DIR = File.join(STORAGE_ROOT, "state") -PLAYLISTS_DIR = File.join(STORAGE_ROOT, "playlists") -LOG_DIR = File.join(STATE_DIR, "logs") -LOCK_FILE = File.join(STATE_DIR, "radio.lock") -SUBS_DB_PATH = File.join(STATE_DIR, "subscriptions.db") -EPISODES_DB_PATH = File.join(STATE_DIR, "episodes.db") +CFG = load_config +STATE_DIR = File.join(CFG[:storage], "state") +QUEUE_DIR = File.join(CFG[:storage], "queue") +LOG_DIR = File.join(CFG[:storage], "logs") +SUBS_DB = File.join(STATE_DIR, "subscriptions.db") +PLAYED_DB = File.join(STATE_DIR, "played.db") -[DIRS_TO_CREATE].each do |d| - Dir.mkdir(d) unless Dir.exist?(d) +MEDIA_DIRS = %w[music podcasts jingles announcements] +MEDIA_DIRS.each do |d| + FileUtils.mkdir_p(File.join(CFG[:storage], d)) end +FileUtils.mkdir_p(STATE_DIR) +FileUtils.mkdir_p(QUEUE_DIR) +FileUtils.mkdir_p(LOG_DIR) + +LOG_FILE = File.join(LOG_DIR, "update_playlists.log") +$log_fh = File.open(LOG_FILE, "a+") -$log_fh = File.open(File.join(LOG_DIR, "update_playlists.log"), "a") def log(level, msg) ts = Time.now.strftime("%Y-%m-%d %H:%M:%S") line = "[#{ts}] [#{level}] #{msg}" - puts(line) - $log_fh.write("#{line}\n") + $stdout.puts(line) + $log_fh.write(line + "\n") $log_fh.flush +rescue Exception => e + # Log file may be unavailable; fall back to stdout only + $stdout.puts("[WARN] Could not write log: #{e.message}") end -db_subs = Sequel.jdbc("sqlite:", SUBS_DB_PATH) -db_eps = Sequel.jdbc("sqlite:", EPISODES_DB_PATH) - -db_subs.execute("PRAGMA journal_mode=WAL;") -db_eps.execute("PRAGMA journal_mode=WAL;") -db_subs.execute("PRAGMA busy_timeout=5000;") -db_eps.execute("PRAGMA busy_timeout=5000;") - -def table_exists?(db, name) - db[:sqlite_master].where(type: "table", name: name).count > 0 +def table_exists?(db, tbl) + db[:sqlite_master].where(type: "table", name: tbl).count > 0 +rescue Exception => e + false end -def ensure_schema(db, path, label) - if !table_exists?(db, "shows") - db.execute <<-SQL -CREATE TABLE IF NOT EXISTS shows ( - guid TEXT PRIMARY KEY, - slug TEXT UNIQUE NOT NULL, - title TEXT NOT NULL, - feed_url TEXT NOT NULL, - audio_only INTEGER DEFAULT 1, - archive INTEGER DEFAULT 0, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -) +def ensure_schema(db_subs, db_eps) + unless table_exists?(db_subs, :shows) + db_subs.execute <<-SQL + CREATE TABLE IF NOT EXISTS shows ( + guid TEXT PRIMARY KEY, + slug TEXT UNIQUE NOT NULL, + title TEXT NOT NULL, + feed_url TEXT NOT NULL, + description TEXT DEFAULT '', + category TEXT DEFAULT '', + audio_only INTEGER DEFAULT 1, + archive INTEGER DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) SQL - log("INFO", "Created 'shows' table in #{label}") end - - if !table_exists?(db, "episodes") - db.execute <<-SQL -CREATE TABLE IF NOT EXISTS episodes ( - guid TEXT PRIMARY KEY, - show_guid TEXT NOT NULL REFERENCES shows(guid), - title TEXT, - enclosure_url TEXT, - runlength INTEGER DEFAULT 0, - played INTEGER DEFAULT 0, - downloaded INTEGER DEFAULT 0, - local_path TEXT, - fetched_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP -) + unless table_exists?(db_eps, :episodes) + db_eps.execute <<-SQL + CREATE TABLE IF NOT EXISTS episodes ( + guid TEXT PRIMARY KEY, + show_guid TEXT NOT NULL, + title TEXT NOT NULL, + enclosure_url TEXT NOT NULL, + enclosure_type TEXT DEFAULT '', + duration_sec INTEGER DEFAULT 0, + pub_date TEXT DEFAULT '', + local_path TEXT DEFAULT '', + played INTEGER DEFAULT 0, + downloaded INTEGER DEFAULT 0, + fetched_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) SQL - log("INFO", "Created 'episodes' table in #{label}") end end -ensure_schema(db_subs, SUBS_DB_PATH, "subscriptions.db") -ensure_schema(db_eps, EPISODES_DB_PATH, "episodes.db") - -def acquire_lock! - @lock_fh = File.new(LOCK_FILE, "a+") - begin - @lock_fh.flock(File::LOCK_EX | File::LOCK_NB) - rescue IOError - log("WARN", "Another radio process holds the lock; skipping this run.") - exit 0 - end -end - -def release_lock! - @lock_fh.flock(File::LOCK_UN) if @lock_fh - @lock_fh.close if @lock_fh - @lock_fh = nil -end - -def make_annotated_uri(ep) - uri = ep[:local_path] || ep[:enclosure_url] - rl = ep[:runlength].to_i - ttl = ep[:title].to_s.gsub('"', '\\"') - "annotate:liq_runlength=\"#{rl}\",liq_title=\"#{ttl}\":#{uri}" -end - -def select_next_episode(show_guid) - # Try to find an unplayed episode - ep = db_eps[:episodes].where(show_guid: show_guid, played: 0).order(:fetched_at.asc).first - - if ep.nil? - # All episodes played — reset all to unplayed, then pick the first - count = db_eps[:episodes].where(show_guid: show_guid).count - if count > 0 - db_eps[:episodes].where(show_guid: show_guid).update(played: 0) - log("INFO", "Reset all episodes for show #{show_guid} to played=0 (cycling)") - ep = db_eps[:episodes].where(show_guid: show_guid).order(:fetched_at.asc).first - end - end - - ep -end - -def update_show_queue(show) - slug = show[:slug] - queue_dir = File.join(PLAYLISTS_DIR, slug) - Dir.mkdir(queue_dir) unless Dir.exist?(queue_dir) - - ep = select_next_episode(show[:guid]) - - if ep.nil? - log("WARN", "No episodes available for show '#{slug}'") - return false - end - - annotated = make_annotated_uri(ep) - queue_file = File.join(queue_dir, "next.uri") - File.write(queue_file, "#{annotated}\n") - - # Mark as played - db_eps[:episodes].where(guid: ep[:guid]).update(played: 1) - - log("INFO", "Queued episode '#{ep[:title]}' for show '#{slug}' (runlength=#{ep[:runlength]}s)") - true -end - -def update_all - shows = db_subs[:shows].all - - if shows.empty? - log("INFO", "No shows registered; nothing to do.") - return - end - - queued_count = 0 - failed_count = 0 - - shows.each do |show| - begin - if update_show_queue(show) - queued_count += 1 - else - failed_count += 1 - end - rescue Exception => e - failed_count += 1 - log("ERROR", "Failed to update queue for show '#{show[:slug]}': #{e.class} #{e.message}") - end - end - - log("INFO", "Update complete: #{queued_count} queued, #{failed_count} failed out of #{shows.size} shows") - - if ARGV.include?("--json") - summary = {} - shows.each do |show| - total = db_eps[:episodes].where(show_guid: show[:guid]).count - played = db_eps[:episodes].where(show_guid: show[:guid], played: 1).count - unplayed = total - played - summary[show[:slug]] = { total: total, played: played, unplayed: unplayed } - end - puts(JSON.pretty_generate(summary)) - end -end +$stdout.sync = true begin - acquire_lock! - begin - update_all - ensure - release_lock! + db_subs = Sequel.connect("jdbc:sqlite:" + SUBS_DB) + db_eps = Sequel.connect("jdbc:sqlite:" + PLAYED_DB) +ensure_schema(db_subs, db_eps) + + json_output = ARGV.include?("--json") + results = [] + + shows = db_subs[:shows].all + if shows.empty? + log("INFO", "No shows registered.") + else + shows.each do |show| + slug = show[:slug] + show_g = show[:guid] + archive = show[:archive] || 0 + + total = db_eps[:episodes].where(show_guid: show_g).count + unplayed = db_eps[:episodes].where(show_guid: show_g, played: 0).count + + chosen = nil + + if total == 0 + log("INFO", "#{slug}: no episodes yet, SKIP") + elsif unplayed == 0 && archive == 0 + # Cycle: reset all to unplayed, pick first, mark played + db_eps[:episodes].where(show_guid: show_g).update(played: 0) + ep = db_eps[:episodes].where(show_guid: show_g).order(:pub_date.asc).first + if ep + db_eps[:episodes].where(guid: ep[:guid]).update(played: 1) + chosen = ep + log("INFO", "#{slug}: cycled, picked '#{ep[:title]}'") + end + elsif unplayed > 0 + eps = db_eps[:episodes].where(show_guid: show_g, played: 0).all + ep = eps.sample + db_eps[:episodes].where(guid: ep[:guid]).update(played: 1) + chosen = ep + log("INFO", "#{slug}: picked '#{ep[:title]}' (#{unplayed} unplayed)") + end + + qf = File.join(QUEUE_DIR, "#{slug}.txt") + if chosen + dur = chosen[:duration_sec].to_i + annot = "annotate:liq_runlength=\"#{dur}\",liq_title=\"#{chosen[:title]}\"" + uri = chosen[:enclosure_url] + File.write(qf, "#{annot}:#{uri}\n") + else + File.write(qf, "SKIP\n") + end + + results << { + "slug" => slug, + "total" => total, + "unplayed" => unplayed, + "picked" => chosen ? chosen[:title] : nil + } + end end + + if json_output + puts JSON.pretty_generate(results) + end + + log("INFO", "Done. Processed #{results.size} shows.") + + begin + db_subs.disconnect + rescue Exception => e + log("WARN", "Error disconnecting subs: #{e.message}") + end + begin + db_eps.disconnect + rescue Exception => e + log("WARN", "Error disconnecting eps: #{e.message}") + end + $log_fh.close + +rescue Interrupt + log("INFO", "Interrupted.") + exit 1 rescue Exception => e - log("ERROR", "Fatal error: #{e.class} #{e.message}") - log("ERROR", e.backtrace.first(10).join("\n")) + log("ERROR", "#{e.class}: #{e.message}") + log("ERROR", e.backtrace.first(5).join("\n")) exit 1 end -db_subs.disconnect -db_eps.disconnect -$log_fh.close +main if __FILE__ == $PROGRAM_NAME