From efb76d3e198d0cd636322b0b47f58f3285c5cf5c Mon Sep 17 00:00:00 2001 From: Glenn Date: Sun, 5 Feb 2023 16:36:09 +0100 Subject: [PATCH] refactor: massive code clean up --- src/cli.cr | 118 +++---------------------- src/controllers/base_controller.cr | 6 ++ src/controllers/download_controller.cr | 58 ++++++++++++ src/controllers/info_controller.cr | 31 +++++++ src/duration.cr | 37 ++++++++ src/event_page.cr | 73 +++++++++++++++ src/meta_data.cr | 19 ++++ src/utils/url_validator.cr | 25 ++++++ 8 files changed, 260 insertions(+), 107 deletions(-) create mode 100644 src/controllers/base_controller.cr create mode 100644 src/controllers/download_controller.cr create mode 100644 src/controllers/info_controller.cr create mode 100644 src/duration.cr create mode 100644 src/event_page.cr create mode 100644 src/meta_data.cr create mode 100644 src/utils/url_validator.cr diff --git a/src/cli.cr b/src/cli.cr index 0be31c6..0b7feed 100644 --- a/src/cli.cr +++ b/src/cli.cr @@ -1,6 +1,11 @@ require "option_parser" +require "./duration" +require "./meta_data" +require "./controllers/info_controller" +require "./controllers/download_controller" + # Cli part of FosdemRecorder module FosdemRecorder # Fosdem Cli - Download and cut streams video @@ -22,26 +27,7 @@ module FosdemRecorder Cli.new(args) end - private def info(url) - _validate_url(url) - meta = _get_meta(url) - puts meta[:title].colorize.fore(:green) - puts "* event start = #{meta[:event_start]}" - puts "* event stop = #{meta[:event_stop]}" - puts "* event length = #{meta[:duration_str]}" - puts "* stream url = #{meta[:stream_url]}" - end - private def download(url) - _validate_url(url) - meta = _get_meta(url) - - localtime = meta[:event_start].to_local - timeformat = localtime.to_s("%H:%M %Y-%m-%d") - cmd = "echo ffmpeg -i #{meta[:stream_url]} -c copy -t #{meta[:duration_str]} \"#{meta[:title_sane]}.mp4\" | at #{timeformat}" - puts "Command: #{cmd}".colorize.fore(:yellow) - system cmd - end private def parse(args) commands = [] of Proc(String, Nil) @@ -49,12 +35,16 @@ module FosdemRecorder OptionParser.parse(args) do |opts| opts.banner = "Usage: fosdem-recorder [subcommand] [arguments]" + # opts.on("update-cache", "Fetch schedule information") do + # commands << ->InfoController.process(String) + # end + opts.on("info", "Get information about URL") do - commands << ->info(String) + commands << ->InfoController.process(String) end opts.on("download", "Download conference described at URL") do - commands << ->download(String) + commands << ->DownloadController.process(String) end opts.on("-h", "--help", "Shows this help") do @@ -69,91 +59,5 @@ module FosdemRecorder end end - private def _validate_url(url) - return if url =~ %r{^https://fosdem.org/\d+/schedule/event/.*} - - if url =~ %r{^https://fosdem.org/.*} - STDERR.puts "ERROR: not a schedule page. URL must contain .../schedule/event/..." - exit 1 - end - - STDERR.puts "ERROR: not a fosdem stream. URL must start with https://fosdem.org/..." - exit 1 - end - - private def _get_meta(url) - puts "Loading data from #{url}".colorize.fore(:yellow) - mechanize = Mechanize.new - - begin - page = mechanize.get(url) - rescue ex : Socket::Addrinfo::Error - STDERR.puts "ERROR: #{ex.message}" - exit 1 - end - - # body_class = page.at('body').attr('class') - # if body_class != 'schedule-event' - # STDERR.puts "ERROR: Not an event schedule page!" - # exit 1 - # end - # puts body_class - - title = page.title - title_sane = - title - .gsub(/[^a-zA-Z0-9]/, "-") - .gsub(/--*/, "-") - .gsub(/-$/, "") - .gsub(/^-/, "") - - play_start_str = - page - .css(".side-box .icon-play").first.parent - .try &.css(".value-title").first["title"].strip - play_start_str = "" if play_start_str.nil? - - puts "PLAY_START = #{play_start_str}" - location = Time::Location.load("Europe/Brussels") - # play_start = Time.parse(play_start_str, "%H:%S", location) - play_start = Time.parse_rfc3339(play_start_str) #, location) - - play_stop_str = - page - .css(".side-box .icon-stop").first.parent - .try &.css(".value-title").first["title"].strip - play_stop_str = "" if play_stop_str.nil? - - # play_stop = Time.parse(play_stop_str, "%H:%S", location) - play_stop = Time.parse_rfc3339(play_stop_str) - - duration = (play_stop - play_start).to_i / 3600 - duration_h = duration.to_i - duration_m = ((duration - duration_h) * 60 + 1).to_i - duration_str = sprintf("%02d:%02d:00", { duration_h, duration_m }) - - stream_page = - page - .links - .select { |link| link.href =~ /live.fosdem.org/ } - .first - .href - - stream_url = - stream_page - .gsub(%r{.*watch/}, "https://stream.fosdem.org/") - .gsub(/$/, ".m3u8") - - { - title: title, - title_sane: title_sane, - stream_url: stream_url, - event_start: play_start, - event_stop: play_stop, - event_start_str: play_start_str, - event_stop_str: play_stop_str, - duration_str: duration_str - } - end end end diff --git a/src/controllers/base_controller.cr b/src/controllers/base_controller.cr new file mode 100644 index 0000000..b6a3475 --- /dev/null +++ b/src/controllers/base_controller.cr @@ -0,0 +1,6 @@ + +module FosdemRecorder + class BaseController + end +end + diff --git a/src/controllers/download_controller.cr b/src/controllers/download_controller.cr new file mode 100644 index 0000000..dad2a02 --- /dev/null +++ b/src/controllers/download_controller.cr @@ -0,0 +1,58 @@ +require "./base_controller" +require "../utils/url_validator" + +module FosdemRecorder + class DownloadController < BaseController + def self.process(url) + UrlValidator.validate_event! url + meta = EventPage.get_meta(url) + + now = Time.local #Time::Location.load("Europe/Brussels") + + event_start = meta.event_start + raise "Event start is missing!" if event_start.nil? + + event_stop = meta.event_stop + raise "Event stop is missing!" if event_stop.nil? + + event_start_localtime = event_start.to_local + postpone_download = (event_start_localtime >= now) + + # compute remaining duration when needed + duration = Duration.new(start: event_start, stop: event_stop) + remaining_duration = duration.from_now + timeformat = event_start_localtime.to_s("%H:%M %Y-%m-%d") + + # FIXME: mark the file as partial + cmd = [ + "ffmpeg", + + # First the stream URL + "-i", meta.stream_url, + + # Then the codec (simple copy) + "-c", "copy", + + # Fix malformed AAC bitstream when detected + "-bsf:a", "aac_adtstoasc", + + # Make the stream playable as we download it + "-movflags", "frag_keyframe+empty_moov+default_base_moof+faststart", + + # Set record duration + "-t", remaining_duration.to_s, + + # Set output filename + "\"#{meta.title_sanitized}.mp4\"" + ].join(" ") + + if postpone_download + cmd = "echo #{cmd} | at #{timeformat}" + else + cmd = "echo #{cmd} | at now" + end + puts "Command: #{cmd}".colorize.fore(:yellow) + system cmd + end + end +end diff --git a/src/controllers/info_controller.cr b/src/controllers/info_controller.cr new file mode 100644 index 0000000..7d84b1a --- /dev/null +++ b/src/controllers/info_controller.cr @@ -0,0 +1,31 @@ + +require "./base_controller" +require "../utils/url_validator" +require "../event_page" + +module FosdemRecorder + class InfoController < BaseController + def self.process(url) + UrlValidator.validate_event! url + meta = EventPage.get_meta(url) + + puts meta.title.colorize.fore(:green) + + start = meta.event_start + stop = meta.event_stop + puts "* event start = #{start}" + puts "* event stop = #{stop}" + + if !start.nil? && !stop.nil? + duration = Duration.new(start: start, stop: stop) + duration_remaining = Duration.new(start: start, stop: stop).from_now + puts "* event length = #{duration.to_s} (from now: #{duration_remaining.to_s})" + else + puts "* event length = (none)" + end + + puts "* stream url = #{meta.stream_url ? meta.stream_url : "(none)"}" + end + end +end + diff --git a/src/duration.cr b/src/duration.cr new file mode 100644 index 0000000..f635b04 --- /dev/null +++ b/src/duration.cr @@ -0,0 +1,37 @@ + +module FosdemRecorder + class Duration + getter start : Time + getter stop : Time + + def initialize(@start, @stop) + end + + def from_now() + Duration.new(start: Time.local, stop: self.stop) + end + + def value() + duration = (self.stop - self.start).to_i + end + + def past? + Time.local > self.stop + end + + def future? + Time.local < self.start + end + + def present? + (Time.local >= self.start) && (Time.local <= self.stop) + end + + def to_s() + duration = self.value / 3600 + duration_h = duration.to_i + duration_m = ((duration - duration_h) * 60 + 1).to_i + duration_str = sprintf("%02d:%02d:00", { duration_h, duration_m }) + end + end +end diff --git a/src/event_page.cr b/src/event_page.cr new file mode 100644 index 0000000..4071ed5 --- /dev/null +++ b/src/event_page.cr @@ -0,0 +1,73 @@ + +module FosdemRecorder + # Fosdem Cli - Download and cut streams video + class EventPage + def self.get_meta(url) + puts "Loading data from #{url}".colorize.fore(:yellow) + mechanize = Mechanize.new + + begin + page = mechanize.get(url) + rescue ex : Socket::Addrinfo::Error + STDERR.puts "ERROR: #{ex.message}" + exit 1 + end + + # body_class = page.at('body').attr('class') + # if body_class != 'schedule-event' + # STDERR.puts "ERROR: Not an event schedule page!" + # exit 1 + # end + # puts body_class + + title = page.title + title_sanitized = + title + .gsub(/[^a-zA-Z0-9]/, "-") + .gsub(/--*/, "-") + .gsub(/-$/, "") + .gsub(/^-/, "") + + play_start_str = + page + .css(".side-box .icon-play").first.parent + .try &.css(".value-title").first["title"].strip + play_start_str = "" if play_start_str.nil? + + location = Time::Location.load("Europe/Brussels") + # play_start = Time.parse(play_start_str, "%H:%S", location) + play_start = Time.parse_rfc3339(play_start_str) #, location) + + play_stop_str = + page + .css(".side-box .icon-stop").first.parent + .try &.css(".value-title").first["title"].strip + play_stop_str = "" if play_stop_str.nil? + + # play_stop = Time.parse(play_stop_str, "%H:%S", location) + play_stop = Time.parse_rfc3339(play_stop_str) + + + stream_page = + page + .links + .select { |link| link.href =~ /live.fosdem.org/ } + .first? + .try &.href + + stream_url = + stream_page + .try &.gsub(%r{.*watch/}, "https://stream.fosdem.org/") + .gsub(/$/, ".m3u8") + + + meta = MetaData.new( + title: title, + title_sanitized: title_sanitized, + stream_url: stream_url, + event_start: play_start, + event_stop: play_stop, + ) + end + end +end diff --git a/src/meta_data.cr b/src/meta_data.cr new file mode 100644 index 0000000..40d68f5 --- /dev/null +++ b/src/meta_data.cr @@ -0,0 +1,19 @@ + +module FosdemRecorder + class MetaData + getter title : String? + getter title_sanitized : String? + getter stream_url : String? + + getter event_start : Time? + getter event_stop : Time? + + def initialize( + @title, + @title_sanitized, + @stream_url, + @event_start, + @event_stop) + end + end +end diff --git a/src/utils/url_validator.cr b/src/utils/url_validator.cr new file mode 100644 index 0000000..7704783 --- /dev/null +++ b/src/utils/url_validator.cr @@ -0,0 +1,25 @@ + + +module FosdemRecorder + class UrlValidator + class PageIsNotFosdemError < Exception + def message + "Not a fosdem stream. URL must start with https://fosdem.org/..." + end + end + + class PageIsNotScheduleError < Exception + def message + "Not a schedule page. URL must contain .../schedule/event/..." + end + end + + def self.validate_event!(url) : Nil + return if url =~ %r{^https://fosdem.org/\d+/schedule/event/.*} + + raise PageIsNotScheduleError.new if url =~ %r{^https://fosdem.org/.*} + raise PageIsNotFosdemError.new + nil + end + end +end