Merge branch 'develop' of code.apps.glenux.net:glenux/code-preloader into develop
Some checks failed
continuous-integration/drone/push Build is failing

This commit is contained in:
Glenn Y. Rolland 2024-01-04 22:55:33 +01:00
commit a53ae57f51
9 changed files with 209 additions and 123 deletions

View file

@ -12,10 +12,10 @@ ignore_list:
- ^Makefile
- .*\.svg$
output_file_path: null
header_prompt_file_path: null
footer_prompt_file_path: null
output_path: null
prompt:
header_path: null
footer_path: null
template_path: misc/templates/default.j2
#

View file

@ -10,10 +10,10 @@ ignore_list:
- prompts
- Makefile
output_file_path: null
output_path: null
header_prompt_file_path: null
footer_prompt_file_path: prompts/footer.txt
prompt:
header_path: null
footer_path: prompts/footer.txt
#

16
misc/templates/default.j2 Normal file
View file

@ -0,0 +1,16 @@
{%- if prompt_header -%}
@@ CONTEXT
{{ prompt_header }}
{%- endif -%}
{%- for file in prompt_files -%}
@@ FILE "{{ file.path }}" WITH MIME-TYPE "{{ file.mime_type }}"
{{- file.content -}}
{%- endfor -%}
{%- if prompt_footer -%}
@@ REQUEST
{{ prompt_footer }}
{%- endif -%}

View file

@ -1,10 +1,8 @@
# vim: set ts=2 sw=2 et ft=crystal:
require "colorize"
require "file"
require "option_parser"
require "magic"
require "crinja"
require "./config"
require "./filelist"
@ -12,14 +10,16 @@ require "./filelist"
# The CodePreloader module organizes classes and methods related to preloading code files.
module CodePreloader
# The Cli class handles command-line interface operations for the CodePreloader.
class Cli
alias ProcessedFile = NamedTuple(path: String, content: String, mime_type: String)
@config : Config
# Initializes the Cli class with default values.
def initialize(args)
@output_file_path = ""
@output_path = ""
@config = Config.new()
@config.detect_config()
@config.parse_arguments(args)
end
@ -43,7 +43,7 @@ module CodePreloader
default_config_path = "example.code_preloader.yml"
# Use the specified path if provided, otherwise use the default
config_file_path = init_options.config_file_path || default_config_path
config_path = init_options.config_path || default_config_path
# Content of the .code_preloader.yml file
config_content = [
@ -51,7 +51,7 @@ module CodePreloader
"# Example configuration for Code-Preloader",
"",
"# List of repository paths to preload",
"# repository_path_list:",
"# source_list:",
"# - \"path/to/repo1\"",
"# - \"path/to/repo2\"",
"",
@ -60,19 +60,19 @@ module CodePreloader
" - ^\\.git/.*",
"",
"# Path to the output file (if null, output to STDOUT)",
"output_file_path: null",
"output_path: null",
"",
"# Optional: Path to a file containing the header prompt",
"header_prompt_file_path: null",
"header_path: null",
"",
"# Optional: Path to a file containing the footer prompt",
"footer_prompt_file_path: null",
"footer_path: null",
""
].join("\n")
# Writing the configuration content to the file
File.write(config_file_path, config_content)
puts "Configuration file created at: #{config_file_path}"
File.write(config_path, config_content)
puts "Configuration file created at: #{config_path}"
rescue e : Exception
abort("ERROR: Unable to create the configuration file: #{e.message}")
end
@ -88,7 +88,9 @@ module CodePreloader
end
def exec_help
puts @config.parser
@config.help_options.try do |opts|
puts opts.parser_snapshot
end
exit(0)
end
@ -96,62 +98,73 @@ module CodePreloader
abort("Unexpected nil value for pack_options!") if pack_options.nil?
preloaded_content = {} of String => NamedTuple(mime: String, content: String)
output_file_path = pack_options.output_file_path
repository_path_list = pack_options.repository_path_list
header_prompt_file_path = pack_options.header_prompt_file_path
footer_prompt_file_path = pack_options.footer_prompt_file_path
config_path = pack_options.config_path
output_path = pack_options.output_path
source_list = pack_options.source_list
prompt_header_path = pack_options.prompt_header_path
prompt_footer_path = pack_options.prompt_footer_path
prompt_template_path = pack_options.prompt_template_path
regular_output_file = false
header_prompt = ""
footer_prompt = ""
prompt_header_content = nil
prompt_footer_content = nil
prompt_template_content = ""
STDERR.puts "Loading config file from: #{config_path}".colorize(:yellow)
filelist = FileList.new()
filelist.add(repository_path_list)
filelist.add(source_list)
pack_options.ignore_list.each do |ignore_pattern|
filelist.reject { |path| !!(path =~ Regex.new(ignore_pattern)) }
end
if !header_prompt_file_path.nil?
STDERR.puts "Loading header prompt from: #{header_prompt_file_path}".colorize(:yellow)
header_prompt = File.read(header_prompt_file_path)
abort("No prompt file defined!") if prompt_template_path.nil?
prompt_template_content = File.read(prompt_template_path)
if !prompt_header_path.nil?
STDERR.puts "Loading header prompt from: #{prompt_header_path}".colorize(:yellow)
prompt_header_content = File.read(prompt_header_path)
end
if !footer_prompt_file_path.nil?
STDERR.puts "Loading footer prompt from: #{footer_prompt_file_path}".colorize(:yellow)
footer_prompt = File.read(footer_prompt_file_path)
if !prompt_footer_path.nil?
STDERR.puts "Loading footer prompt from: #{prompt_footer_path}".colorize(:yellow)
prompt_footer_content = File.read(prompt_footer_path)
end
output_file = STDOUT
output_file_path.try do |path|
output_path.try do |path|
break if path.empty?
break if path == "-"
regular_output_file = true
output_file = File.open(path, "w")
end
STDERR.puts "Writing output to: #{regular_output_file ? output_file_path : "stdout" }".colorize(:yellow)
STDERR.puts "Writing output to: #{regular_output_file ? output_path : "stdout" }".colorize(:yellow)
# FIXME: prompt_header_path.try { output_file.puts prompt_header_content }
header_prompt_file_path.try { output_file.puts header_prompt }
STDERR.puts "Processing repository: #{repository_path_list}".colorize(:yellow)
STDERR.puts "Processing source directories: #{source_list}".colorize(:yellow)
processed_files = [] of ProcessedFile
filelist.each do |file_path|
STDERR.puts "Processing file: #{file_path}".colorize(:yellow)
file_result = process_file(file_path, output_file)
output_file.puts "@@ File \"#{file_path}\" (Mime-Type: #{file_result[:mime]})"
output_file.puts ""
if file_result[:text_content] !~ /^\s*$/
output_file.puts(file_result[:text_content])
output_file.puts ""
end
processed_files << file_result
end
footer_prompt_file_path.try { output_file.puts footer_prompt }
# FIXME: prompt_footer_path.try { output_file.puts prompt_footer_content }
output_file.puts Crinja.render(
prompt_template_content,
{
"prompt_header": prompt_header_content,
"prompt_files": processed_files,
"prompt_footer": prompt_footer_content
}
)
output_file.close if regular_output_file
STDERR.puts "Processing completed.".colorize(:yellow)
rescue e : Exception
STDERR.puts "An error occurred during execution: #{e.message}"
STDERR.puts "ERROR: #{e.message}"
exit(1)
end
@ -168,8 +181,9 @@ module CodePreloader
end
return {
mime: mime,
text_content: clean_content
path: file_path,
content: clean_content,
mime_type: mime
}
end
end

View file

@ -16,23 +16,30 @@ module CodePreloader
Version
end
class HelpOptions
property parser_snapshot : OptionParser? = nil
end
class InitOptions
property config_file_path : String? = nil
property config_path : String? = nil
end
class PackOptions
property config_file_path : String? = nil
property repository_path_list : Array(String) = [] of String
property config_path : String? = nil
property source_list : Array(String) = [] of String
property ignore_list : Array(String) = [] of String
property output_file_path : String?
property header_prompt_file_path : String?
property footer_prompt_file_path : String?
property output_path : String?
property prompt_template_path : String?
property prompt_header_path : String?
property prompt_footer_path : String?
end
getter verbose : Bool = false
getter parser : OptionParser?
property subcommand : Subcommand = Subcommand::None
property pack_options : PackOptions?
property init_options : InitOptions?
getter subcommand : Subcommand = Subcommand::None
getter pack_options : PackOptions?
getter init_options : InitOptions?
getter help_options : HelpOptions?
def initialize()
end
@ -49,7 +56,7 @@ module CodePreloader
parser.unknown_args do |remaining_args, _|
# FIXME: detect and make error if there are more or less than one
remaining_args.each do |arg|
@init_options.try &.config_file_path = arg
@init_options.try &.config_path = arg
end
end
@ -58,7 +65,7 @@ module CodePreloader
"--config=FILE",
"Load parameters from FILE"
) do |config_file|
@init_options.try { |opt| opt.config_file_path = config_file }
@init_options.try { |opt| opt.config_path = config_file }
end
parser.separator ""
@ -79,16 +86,44 @@ module CodePreloader
def parse_pack_options(parser)
@pack_options = PackOptions.new
config_file = detect_config_file
config_file.try { |path| load_pack_config(path) }
parser.banner = [
"Usage: code-preloader pack [options] DIR ...\n",
"Global options:"
].join("\n")
parser.separator "\nPack options:"
parser.on(
"-c FILE",
"--config=FILE",
"Load parameters from FILE\n(default: \".code_preload.yml\", if present)"
) do |config_file|
@pack_options.try { |opt| load_pack_config(config_file) }
end
parser.on(
"-F FILE",
"--prompt-footer=FILE",
"Load prompt footer from FILE (default: none)"
) do |prompt_footer_path|
@pack_options.try { |opt| opt.prompt_footer_path = prompt_footer_path }
end
parser.on(
"-H FILE",
"--prompt-header=FILE",
"Load prompt header from FILE (default: none)"
) do |prompt_header_path|
@pack_options.try { |opt| opt.prompt_header_path = prompt_header_path }
end
parser.on(
"-i REGEXP",
"--ignore=REGEXP",
"Ignore file or directory"
"Ignore file or directory. Can be used\nmultiple times (default: none)"
) do |ignore_file|
@pack_options.try { |opt| opt.ignore_list << ignore_file }
end
@ -96,40 +131,24 @@ module CodePreloader
parser.on(
"-o FILE",
"--output=FILE",
"Write output to FILE"
"Write output to FILE (default: \"-\", STDOUT)"
) do |output_file|
@pack_options.try { |opt| opt.output_file_path = output_file }
@pack_options.try { |opt| opt.output_path = output_file }
end
parser.on(
"-H FILE",
"--header-prompt=FILE",
"Load header prompt from FILE"
) do |header_prompt_file|
@pack_options.try { |opt| opt.header_prompt_file_path = header_prompt_file }
end
parser.on(
"-F FILE",
"--footer-prompt=FILE",
"Load footer prompt from FILE"
) do |footer_prompt_file|
@pack_options.try { |opt| opt.footer_prompt_file_path = footer_prompt_file }
end
parser.on(
"-c FILE",
"--config=FILE",
"Load parameters from FILE"
) do |config_file|
@pack_options.try { |opt| load_pack_config(config_file) }
"-t FILE",
"--template=FILE",
"Load template from FILE (default: internal)"
) do |prompt_template_path|
@pack_options.try { |opt| opt.prompt_template_path = prompt_template_path }
end
parser.separator ""
parser.unknown_args do |remaining_args, _|
remaining_args.each do |arg|
@pack_options.try { |opt| opt.repository_path_list << arg }
@pack_options.try { |opt| opt.source_list << arg }
end
end
@ -153,13 +172,22 @@ module CodePreloader
"Global options:"
].join("\n")
parser.on("-h", "--help", "Show this help") do
@subcommand = Subcommand::Help
@help_options = HelpOptions.new
@help_options.try do |opts|
opts.parser_snapshot = parser.dup
end
end
parser.on("-v", "--verbose", "Enable verbose mode") do
@verbose = true
end
parser.on("--version", "Show version") do
@subcommand = Subcommand::Version
end
parser.on("-h", "--help", "Show this help") do
@subcommand = Subcommand::Help
end
parser.separator "\nSubcommands:"
@ -187,8 +215,24 @@ module CodePreloader
validate
end
def detect_config
# FIXME: detect config name, if any
def detect_config_file() : String?
home_dir = ENV["HOME"]
possible_files = [
File.join(".code_preloader.yaml"),
File.join(".code_preloader.yml"),
File.join(home_dir, ".config", "code_preloader", "config.yaml"),
File.join(home_dir, ".config", "code_preloader", "config.yml"),
File.join(home_dir, ".config", "code_preloader.yaml"),
File.join(home_dir, ".config", "code_preloader.yml"),
File.join("/etc", "code_preloader", "config.yaml"),
File.join("/etc", "code_preloader", "config.yml"),
]
possible_files.each do |file_path|
return file_path if File.exists?(file_path)
end
return nil
end
private def validate
@ -209,39 +253,42 @@ module CodePreloader
private def validate_pack
opts = @pack_options
abort("No pack options defined!") if opts.nil?
abort("Missing repository path.") if opts.repository_path_list.empty?
abort("Missing repository path.") if opts.source_list.empty?
end
# Reads and returns a list of paths to ignore from the given file.
def self.get_ignore_list(ignore_file_path : String) : Array(String)
File.exists?(ignore_file_path) ? File.read_lines(ignore_file_path).map(&.strip) : [] of String
def self.get_ignore_list(ignore_path : String) : Array(String)
File.exists?(ignore_path) ? File.read_lines(ignore_path).map(&.strip) : [] of String
rescue e : IO::Error
STDERR.puts "Error reading ignore file: #{e.message}"
exit(1)
end
private def load_pack_config(config_file_path : String)
private def load_pack_config(config_path : String)
opts = @pack_options
abort("FIXME") if opts.nil?
abort("No pack options defined!") if opts.nil?
config_str = File.read(config_file_path)
config_str = File.read(config_path)
root = Models::RootConfig.from_yaml(config_str)
opts.config_file_path = config_file_path
if opts.repository_path_list.nil? || opts.repository_path_list.try &.empty?
root.repository_path_list.try { |value| opts.repository_path_list = value }
opts.config_path = config_path
if opts.source_list.nil? || opts.source_list.try &.empty?
root.source_list.try { |value| opts.source_list = value }
end
if opts.ignore_list.nil? || opts.ignore_list.try &.empty?
root.ignore_list.try { |value| opts.ignore_list = value }
end
if opts.output_file_path.nil?
opts.output_file_path = root.output_file_path
if opts.output_path.nil?
opts.output_path = root.output_path
end
if opts.header_prompt_file_path.nil?
root.header_prompt_file_path.try { |value| opts.header_prompt_file_path = value }
if opts.prompt_header_path.nil?
root.prompt.try &.header_path.try { |value| opts.prompt_header_path = value }
end
if opts.footer_prompt_file_path.nil?
root.footer_prompt_file_path.try { |value| opts.footer_prompt_file_path = value }
if opts.prompt_footer_path.nil?
root.prompt.try &.footer_path.try { |value| opts.prompt_footer_path = value }
end
if opts.prompt_template_path.nil?
root.prompt.try &.template_path.try { |value| opts.prompt_template_path = value }
end
rescue ex : Exception

View file

@ -2,10 +2,8 @@
require "walk"
module CodePreloader
# Manage a list of files
class FileList
alias Filter = String -> Bool
class NotADirectory < Exception

View file

@ -1,10 +1,5 @@
# vim: set ts=2 sw=2 et ft=crystal:
require "./cli"
# Now that we have checked for nil, it's safe to use not_nil!
app = CodePreloader::Cli.new(ARGV)
app.exec()

View file

@ -0,0 +1,18 @@
require "yaml"
module CodePreloader::Models
class PromptConfig
include YAML::Serializable
include YAML::Serializable::Strict
@[YAML::Field(key: "header_path")]
getter header_path : String?
@[YAML::Field(key: "footer_path")]
getter footer_path : String?
@[YAML::Field(key: "template_path")]
getter template_path : String?
end
end

View file

@ -1,22 +1,20 @@
require "yaml"
require "./prompt_config"
module CodePreloader::Models
class RootConfig
include YAML::Serializable
include YAML::Serializable::Strict
@[YAML::Field(key: "repository_path_list")]
getter repository_path_list : Array(String)?
@[YAML::Field(key: "source_list")]
getter source_list : Array(String)?
@[YAML::Field(key: "output_file_path")]
getter output_file_path : String?
@[YAML::Field(key: "output_path")]
getter output_path : String?
@[YAML::Field(key: "header_prompt_file_path")]
getter header_prompt_file_path : String?
@[YAML::Field(key: "footer_prompt_file_path")]
getter footer_prompt_file_path : String?
@[YAML::Field(key: "prompt")]
getter prompt : PromptConfig?
@[YAML::Field(key: "ignore_list")]
getter ignore_list : Array(String)?