feat: improve files filtering
This commit is contained in:
parent
e0f7244db7
commit
fb2b85fcb8
11 changed files with 288 additions and 65 deletions
|
@ -4,3 +4,7 @@ shards:
|
|||
git: https://github.com/dscottboggs/magic.cr.git
|
||||
version: 1.1.0
|
||||
|
||||
walk:
|
||||
git: https://github.com/alexherbo2/walk.cr.git
|
||||
version: 0.1.0+git.commit.765d758c0f966cccc98c1d81c7ccd0c1f71928e3
|
||||
|
||||
|
|
|
@ -12,6 +12,8 @@ authors:
|
|||
dependencies:
|
||||
magic:
|
||||
github: dscottboggs/magic.cr
|
||||
walk:
|
||||
github: alexherbo2/walk.cr
|
||||
|
||||
# description: |
|
||||
# Short description of chatgpt-preloader
|
||||
|
|
10
spec/cli_spec.cr
Normal file
10
spec/cli_spec.cr
Normal file
|
@ -0,0 +1,10 @@
|
|||
|
||||
require "./spec_helper"
|
||||
require "../src/cli"
|
||||
|
||||
describe CodePreloader::Cli do
|
||||
|
||||
it "works" do
|
||||
# false.should eq(true)
|
||||
end
|
||||
end
|
0
spec/config_spec.cr
Normal file
0
spec/config_spec.cr
Normal file
6
spec/filelist_data/alice.txt
Normal file
6
spec/filelist_data/alice.txt
Normal file
|
@ -0,0 +1,6 @@
|
|||
# Alice
|
||||
|
||||
## Who is alice?
|
||||
|
||||
## When is alice?
|
||||
|
7
spec/filelist_data/bob.c
Normal file
7
spec/filelist_data/bob.c
Normal file
|
@ -0,0 +1,7 @@
|
|||
|
||||
#include <stdio.h>
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
printf("Hello world\n");
|
||||
return 0;
|
||||
}
|
104
spec/filelist_spec.cr
Normal file
104
spec/filelist_spec.cr
Normal file
|
@ -0,0 +1,104 @@
|
|||
|
||||
require "./spec_helper"
|
||||
require "../src/cli"
|
||||
|
||||
alias FileList = CodePreloader::FileList
|
||||
|
||||
describe CodePreloader::FileList do
|
||||
|
||||
it "can be created empty" do
|
||||
fl = FileList.new
|
||||
end
|
||||
|
||||
it "can be created with a list of directories" do
|
||||
fl = FileList.new(["src/", "spec/"])
|
||||
end
|
||||
|
||||
it "verifies that initial directories exists" do
|
||||
expect_raises(FileList::NotADirectory) do
|
||||
fl = FileList.new(["Alice", "Bob"])
|
||||
end
|
||||
end
|
||||
|
||||
it "can append extra sources" do
|
||||
fl = FileList.new()
|
||||
fl.add "spec/"
|
||||
end
|
||||
|
||||
it "verifies that appended directories exists" do
|
||||
fl = FileList.new()
|
||||
expect_raises(FileList::NotADirectory) do
|
||||
fl.add "Alice"
|
||||
end
|
||||
end
|
||||
|
||||
it "accept adding reject filters" do
|
||||
fl = FileList.new()
|
||||
fl.reject { |item| !!(item =~ /name/) }
|
||||
end
|
||||
|
||||
it "accept adding select filters" do
|
||||
fl = FileList.new()
|
||||
fl.select { |item| !!(item =~ /name/) }
|
||||
end
|
||||
|
||||
it "enumerates the files" do
|
||||
fl = FileList.new()
|
||||
fl.add("spec/filelist_data")
|
||||
|
||||
files = Dir["spec/filelist_data/*"]
|
||||
fl.each do |file|
|
||||
files.should contain(file)
|
||||
files = files - [file]
|
||||
end
|
||||
files.size.should eq(0)
|
||||
end
|
||||
|
||||
it "doesn't enumerate duplicate files" do
|
||||
fl = FileList.new()
|
||||
fl.add("spec/filelist_data")
|
||||
fl.add("spec/filelist_data")
|
||||
|
||||
files = [] of String
|
||||
fl.each do |file|
|
||||
files << file
|
||||
end
|
||||
files.size.should eq(files.uniq.size)
|
||||
end
|
||||
|
||||
it "doesn't enumerate files filtered out by select" do
|
||||
fl = FileList.new()
|
||||
fl.add("spec/filelist_data")
|
||||
fl.select { |path| !!(path =~ /\.c$/) }
|
||||
|
||||
files = Dir["spec/filelist_data/*.c"]
|
||||
fl.each do |file|
|
||||
files.should contain(file)
|
||||
files = files - [file]
|
||||
end
|
||||
files.size.should eq(0)
|
||||
end
|
||||
|
||||
it "doesn't enumerate files filtered out by reject" do
|
||||
fl = FileList.new()
|
||||
fl.add("spec/filelist_data")
|
||||
fl.reject { |path| !!(path =~ /\.txt$/) }
|
||||
|
||||
files = Dir["spec/filelist_data/*.c"]
|
||||
fl.each do |file|
|
||||
files.should contain(file)
|
||||
files = files - [file]
|
||||
end
|
||||
files.size.should eq(0)
|
||||
|
||||
end
|
||||
|
||||
it "export the files as an array" do
|
||||
end
|
||||
|
||||
it "doesn't export duplicate files" do
|
||||
end
|
||||
|
||||
it "doesn't export filtered out files" do
|
||||
end
|
||||
end
|
4
spec/spec_helper.cr
Normal file
4
spec/spec_helper.cr
Normal file
|
@ -0,0 +1,4 @@
|
|||
|
||||
require "spec"
|
||||
|
||||
# require "../src/"
|
84
src/cli.cr
84
src/cli.cr
|
@ -6,6 +6,7 @@ require "option_parser"
|
|||
require "magic"
|
||||
|
||||
require "./config"
|
||||
require "./filelist"
|
||||
|
||||
# The CodePreloader module organizes classes and methods related to preloading code files.
|
||||
module CodePreloader
|
||||
|
@ -20,90 +21,61 @@ module CodePreloader
|
|||
@config.parse_arguments(args)
|
||||
end
|
||||
|
||||
|
||||
# Executes the main functionality of the CLI application.
|
||||
def exec
|
||||
header_prompt = ""
|
||||
footer_prompt = ""
|
||||
__header_prompt_file_path = @config.header_prompt_file_path
|
||||
__footer_prompt_file_path = @config.footer_prompt_file_path
|
||||
__output_file_path = @output_file_path
|
||||
__repository_path_list = @config.repository_path_list
|
||||
# get local values for typing
|
||||
output_file_path = @output_file_path
|
||||
repository_path_list = @config.repository_path_list
|
||||
header_prompt_file_path = @config.header_prompt_file_path
|
||||
footer_prompt_file_path = @config.footer_prompt_file_path
|
||||
|
||||
if !__header_prompt_file_path.nil?
|
||||
STDERR.puts "Loading header prompt from: #{__header_prompt_file_path}"
|
||||
header_prompt = File.read(__header_prompt_file_path)
|
||||
filelist = FileList.new()
|
||||
filelist.add(repository_path_list)
|
||||
@config.ignore_list.each do |ignore_pattern|
|
||||
filelist.reject { |path| !!(path =~ Regex.new(ignore_pattern)) }
|
||||
end
|
||||
|
||||
if !__footer_prompt_file_path.nil?
|
||||
STDERR.puts "Loading footer prompt from: #{__footer_prompt_file_path}"
|
||||
footer_prompt = File.read(__footer_prompt_file_path)
|
||||
if !header_prompt_file_path.nil?
|
||||
STDERR.puts "Loading header prompt from: #{header_prompt_file_path}"
|
||||
header_prompt = File.read(header_prompt_file_path)
|
||||
end
|
||||
|
||||
if !footer_prompt_file_path.nil?
|
||||
STDERR.puts "Loading footer prompt from: #{footer_prompt_file_path}"
|
||||
footer_prompt = File.read(footer_prompt_file_path)
|
||||
end
|
||||
|
||||
abort("@output_file_path should be non-nil here") if __output_file_path.nil?
|
||||
abort("@repository_path should be non-empty here") if __repository_path_list.empty?
|
||||
unless output_file_path.nil? || output_file_path.try(&.empty?) || (output_file_path != "-")
|
||||
output_file = File.open(output_file_path, "w")
|
||||
invalid_output_file = false
|
||||
end
|
||||
|
||||
invalid_output_file = true
|
||||
output_file = STDOUT
|
||||
|
||||
unless __output_file_path.nil? || __output_file_path.try(&.empty?) || (__output_file_path != "-")
|
||||
output_file = File.open(__output_file_path, "w")
|
||||
invalid_output_file = false
|
||||
end
|
||||
header_prompt = ""
|
||||
footer_prompt = ""
|
||||
|
||||
output_file.puts header_prompt if @config.header_prompt_file_path
|
||||
|
||||
STDERR.puts "Processing repository: #{@config.repository_path_list}"
|
||||
__repository_path_list.each do |repository_path|
|
||||
process_repository(repository_path, output_file)
|
||||
filelist.each do |file_path|
|
||||
process_file(file_path, output_file)
|
||||
end
|
||||
|
||||
output_file.puts footer_prompt if @config.footer_prompt_file_path
|
||||
|
||||
output_file.close if !invalid_output_file
|
||||
STDERR.puts "Processing completed. Output written to: #{invalid_output_file ? "stdout" : __output_file_path}"
|
||||
STDERR.puts "Processing completed. Output written to: #{invalid_output_file ? "stdout" : output_file_path}"
|
||||
|
||||
rescue e : Exception
|
||||
STDERR.puts "An error occurred during execution: #{e.message}"
|
||||
exit(1)
|
||||
end
|
||||
|
||||
# Processes the specified repository and writes the output to a file.
|
||||
def process_repository(repository_path : String, output_file : IO::FileDescriptor)
|
||||
process_directory(repository_path, repository_path, output_file)
|
||||
|
||||
rescue e : IO::Error
|
||||
STDERR.puts "Error processing repository: #{e.message}"
|
||||
exit(1)
|
||||
end
|
||||
|
||||
private def process_directory(root_path, dir_path : String, output_file : IO::FileDescriptor)
|
||||
Dir.each_child(dir_path) do |child|
|
||||
child_path = File.join(dir_path, child)
|
||||
|
||||
ignores = (
|
||||
@config.ignore_list
|
||||
.map{ |prefix| [prefix, File.expand_path(child_path) =~ /^#{File.expand_path(prefix)}/] }
|
||||
.reject!{ |item| item[1].nil? }
|
||||
)
|
||||
next if !ignores.empty?
|
||||
|
||||
STDERR.puts "File: #{child_path}"
|
||||
child_path = File.join(dir_path, child)
|
||||
if File.directory?(child_path)
|
||||
process_directory(root_path, child_path, output_file)
|
||||
else
|
||||
process_file(root_path, child_path, output_file)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
private def process_file(root_path : String, file_path : String, output_file : IO::FileDescriptor)
|
||||
relative_file_path = file_path.sub(/^#{Regex.escape(root_path)}/, ".").lstrip
|
||||
private def process_file(file_path : String, output_file : IO::FileDescriptor)
|
||||
fh = File.open(file_path)
|
||||
mime = Magic.mime_type.of(fh)
|
||||
output_file.puts "@@ File \"#{relative_file_path}\" (Mime-Type: #{mime.inspect})"
|
||||
output_file.puts "@@ File \"#{file_path}\" (Mime-Type: #{mime.inspect})"
|
||||
output_file.puts ""
|
||||
output_file.puts(fh.gets_to_end)
|
||||
output_file.puts ""
|
||||
|
|
|
@ -18,23 +18,43 @@ module CodePreloader
|
|||
OptionParser.parse(args) do |parser|
|
||||
parser.banner = "Usage: code-preloader [options] DIR1 ..."
|
||||
|
||||
parser.on("-c CONFIG_FILE", "--config=CONFIG_FILE", "Load parameters from CONFIG_FILE") do |config_file|
|
||||
parser.on(
|
||||
"-c CONFIG_FILE",
|
||||
"--config=CONFIG_FILE",
|
||||
"Load parameters from CONFIG_FILE"
|
||||
) do |config_file|
|
||||
load_config(config_file)
|
||||
end
|
||||
|
||||
parser.on("-i IGNORE_PATH", "--ignore=IGNORE_PATH", "Ignore file or directory") do |ignore_file|
|
||||
parser.on(
|
||||
"-i IGNORE_PATH",
|
||||
"--ignore=IGNORE_PATH",
|
||||
"Ignore file or directory"
|
||||
) do |ignore_file|
|
||||
@ignore_list << ignore_file
|
||||
end
|
||||
|
||||
parser.on("-o OUTPUT_FILE", "--output=OUTPUT_FILE", "Write output to OUTPUT_FILE") do |output_file|
|
||||
parser.on(
|
||||
"-o OUTPUT_FILE",
|
||||
"--output=OUTPUT_FILE",
|
||||
"Write output to OUTPUT_FILE"
|
||||
) do |output_file|
|
||||
@output_file_path = output_file
|
||||
end
|
||||
|
||||
parser.on("-H HEADER_PROMPT_FILE", "--header-prompt=HEADER_PROMPT_FILE", "Load header prompt from HEADER_PROMPT_FILE") do |header_prompt_file|
|
||||
parser.on(
|
||||
"-H HEADER_PROMPT_FILE",
|
||||
"--header-prompt=HEADER_PROMPT_FILE",
|
||||
"Load header prompt from HEADER_PROMPT_FILE"
|
||||
) do |header_prompt_file|
|
||||
@header_prompt_file_path = header_prompt_file
|
||||
end
|
||||
|
||||
parser.on("-F FOOTER_PROMPT_FILE", "--footer-prompt=FOOTER_PROMPT_FILE", "Load footer prompt from FOOTER_PROMPT_FILE") do |footer_prompt_file|
|
||||
parser.on(
|
||||
"-F FOOTER_PROMPT_FILE",
|
||||
"--footer-prompt=FOOTER_PROMPT_FILE",
|
||||
"Load footer prompt from FOOTER_PROMPT_FILE"
|
||||
) do |footer_prompt_file|
|
||||
@footer_prompt_file_path = footer_prompt_file
|
||||
end
|
||||
|
||||
|
@ -50,12 +70,12 @@ module CodePreloader
|
|||
end
|
||||
end
|
||||
|
||||
validate_arguments
|
||||
validate
|
||||
end
|
||||
|
||||
private def validate_arguments
|
||||
private def validate
|
||||
abort("Missing repository path.") if @repository_path_list.empty?
|
||||
abort("Missing repository path.") if
|
||||
|
||||
STDERR.puts("Output file path not specified (using STDOUT)") if @output_file_path.nil? || @output_file_path.try(&.empty?)
|
||||
end
|
||||
|
||||
|
@ -78,7 +98,7 @@ module CodePreloader
|
|||
@header_prompt_file_path = root.header_prompt_file_path || @header_prompt_file_path
|
||||
@footer_prompt_file_path = root.footer_prompt_file_path || @footer_prompt_file_path
|
||||
|
||||
rescue ex
|
||||
rescue ex : Exception
|
||||
STDERR.puts "Failed to load config file: #{ex.message}"
|
||||
exit(1)
|
||||
end
|
||||
|
|
94
src/filelist.cr
Normal file
94
src/filelist.cr
Normal file
|
@ -0,0 +1,94 @@
|
|||
|
||||
require "walk"
|
||||
|
||||
module CodePreloader
|
||||
|
||||
# Manage a list of files
|
||||
class FileList
|
||||
|
||||
alias Filter = String -> Bool
|
||||
|
||||
class NotADirectory < Exception
|
||||
def initialize(path)
|
||||
super(path.to_s)
|
||||
end
|
||||
end
|
||||
|
||||
@sources : Array(String)
|
||||
@filters_in : Array(Filter)
|
||||
@filters_out : Array(Filter)
|
||||
|
||||
def initialize(dirs = [] of String)
|
||||
@sources = [] of String
|
||||
@filters_in = [] of Filter
|
||||
@filters_out = [] of Filter
|
||||
dirs.each { |dir| self.add(dir) }
|
||||
end
|
||||
|
||||
def add(dirs : Array(String))
|
||||
dirs.each { |dir| add(dir) }
|
||||
end
|
||||
|
||||
def add(dir : String)
|
||||
raise NotADirectory.new(dir) if !File.exists? dir
|
||||
|
||||
@sources << dir
|
||||
end
|
||||
|
||||
def select(&filter : Filter)
|
||||
@filters_in << filter
|
||||
end
|
||||
|
||||
def reject(&filter : Filter)
|
||||
@filters_out << filter
|
||||
end
|
||||
|
||||
def each(&block)
|
||||
# ensure we display files only once
|
||||
seen = Set(String).new
|
||||
|
||||
# walk each source
|
||||
@sources.each do |dir|
|
||||
walker = Walk::Down.new(dir)
|
||||
|
||||
walker = walker.filter do |path|
|
||||
is_dir = File.directory? path
|
||||
keep = true
|
||||
must_select = false
|
||||
must_reject = false
|
||||
|
||||
@filters_in.each do |filter_in|
|
||||
must_select = must_select || filter_in.call(path.to_s)
|
||||
end
|
||||
keep = keep && must_select if @filters_in.any?
|
||||
keep = keep || is_dir
|
||||
|
||||
@filters_out.each do |filter_out|
|
||||
must_reject = must_reject || filter_out.call(path.to_s)
|
||||
end
|
||||
keep = keep && !must_reject if @filters_out.any?
|
||||
|
||||
keep
|
||||
end
|
||||
|
||||
walker.each do |path|
|
||||
next if File.directory? path
|
||||
|
||||
path = File.realpath(path) if File.symlink? path
|
||||
next if seen.includes? path.to_s
|
||||
|
||||
seen << path.to_s
|
||||
yield path.to_s
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
def to_a()
|
||||
files = [] of String
|
||||
self.each do |path|
|
||||
files << path.to_s
|
||||
end
|
||||
files
|
||||
end
|
||||
end
|
||||
end
|
Loading…
Reference in a new issue