doc: use the new config format

Reviewed-on: #15

.code_preloader.yml

@@ -0,0 +1,21 @@
+---
+
+ignore_list:
+  - ^\.git/.*
+  - ^\.vagrant/.*
+  - ^misc/.*
+  - ^bin/.*
+  - ^lib/.*
+  - ^misc/.*
+  - ^prompts/.*
+  - ^LICENSE
+  - ^Makefile
+  - .*\.svg$
+
+output_path: null
+
+prompt:
+  header_path: null
+  footer_path: null
+  template_path: null
+#

.drone.yml

@@ -19,14 +19,30 @@ steps:
       # Build
       - apk add --update file-dev libmagic-static
       - shards install
-      - shards build --production --static
+      - shards build --error-trace --production --static  # production version
       - strip bin/code-preloader
-      # Verify
+      - ./bin/code-preloader --version  # Simple check
-      - ./bin/code-preloader --version
       # Cache
       - mkdir -p /_cache/bin
       - cp -r bin/code-preloader /_cache/bin/$PACKAGE_BASENAME
 
+  - name: test:spec
+    image: crystallang/crystal:1.10.1-alpine
+    environment:
+      PACKAGE_BASENAME: code-preloader_linux_amd64
+    volumes:
+      - name: cache
+        path: /_cache
+    commands:
+      - pwd
+      # Upgrade alpine to 3.19
+      - sed -i -e 's,alpine/v3.17,alpine/v3.19,' /etc/apk/repositories
+      - apk upgrade --available && sync
+      # Build
+      - apk add --update file-dev libmagic-static
+      - shards install
+      - crystal spec --error-trace
+
   - name: publish:tag
     image: alpine
     environment:

.gitattributes

@@ -0,0 +1 @@
+*.svg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,4 +1,3 @@
-/docs/
 /lib/
 /bin/
 /.shards/

Makefile

@@ -16,7 +16,6 @@ test:
 install:
 	install \
 		-m 755 \
-		-o root \
 		bin/code-preloader \
 		$(PREFIX)/bin
 

README.md

@@ -9,26 +9,34 @@
 ![License LGPL3.0-or-later](https://img.shields.io/badge/license-LGPL3.0--or--later-blue.svg)
 [![Donate on patreon](https://img.shields.io/badge/patreon-donate-orange.svg)](https://patreon.com/glenux)
 
-# Code-Preloader for ChatGPT
+> :information_source: This project is available on our self-hosted server and
+> on CodeBerg and GitHub as mirrors. For the latest updates and comprehensive
+> version of our project, please visit our primary repository at:
+> <https://code.apps.glenux.net/glenux/code-preloader>. 
+
+# Code-Preloader
 
 Code-Preloader is a specialized tool designed to streamline the process of
-working with ChatGPT on coding projects. It preloads and concatenates files
+working on coding projects with interactive large language models (LLM) like
-from a specified directory, allowing for the seamless integration of customized
+ChatGPT, Claude, Mixtral 8x7B, etc. 
-prompts. This tool is ideal for those who seek an alternative to tools like
+
-GitHub Copilot, enabling a more efficient and tailored interaction with
+It preloads and concatenates files from a specified directory, allowing for the
-ChatGPT.
+seamless integration of customized prompts. 
+
+This tool is ideal for those who seek an alternative to tools like GitHub
+Copilot, enabling a tailored interaction with your favorite LLM.
 
 ## Features
 
 * Preload and concatenate files from a given directory.
-* Customizable header and footer prompts for ChatGPT.
+* Customizable header and footer prompts for your LLM.
 * Simple command-line interface for easy operation.
 
 ## Limitations
 
-**Note:** Note: This tool is optimized for smaller codebases. For larger
+**Note:** This tool is optimized for smaller codebases. For larger
 repositories, performance may not be optimal due to processing constraints and
-the nature of interactions with ChatGPT.
+the nature of interactions with LLMs.
 
 ## Prerequisites
 
@@ -66,29 +74,30 @@ git clone https://code.apps.glenux.net/glenux/code-preloader
 cd code-preloader
 make prepare
 make build
-make install
+sudo make install                 # either to install system-wide
+make install PREFIX=$HOME/.local  # or to install as a user
 ```
 
 ## Usage
 
+### Packing directory content
+
 Run Code-Preloader with the following command-line options:
 
 ```
-Usage: code-preloader [options] ROOT_DIR
+Usage: code-preloader pack [options] DIR ...
-    -c CONFIG_FILE, --config=CONFIG_FILE
-                                     Load parameters from CONFIG_FILE
-    -i IGNORE_PATH, --ignore=IGNORE_PATH
-                                     Ignore file or directory
-    -o OUTPUT_FILE, --output=OUTPUT_FILE
-                                     Write output to OUTPUT_FILE
-    -H HEADER_PROMPT_FILE, --header-prompt=HEADER_PROMPT_FILE
-                                     Load header prompt from HEADER_PROMPT_FILE
-    -F FOOTER_PROMPT_FILE, --footer-prompt=FOOTER_PROMPT_FILE
-                                     Load footer prompt from FOOTER_PROMPT_FILE
-    -h, --help                       Show this help
-```
 
-### Examples
+Global options:
+    --version                        Show version
+    -h, --help                       Show this help
+
+Pack options:
+    -i REGEXP, --ignore=REGEXP       Ignore file or directory
+    -o FILE, --output=FILE           Write output to FILE
+    -H FILE, --header-prompt=FILE    Load header prompt from FILE
+    -F FILE, --footer-prompt=FILE    Load footer prompt from FILE
+    -c FILE, --config=FILE           Load parameters from FILE
+```
 
 #### Basic Use Case
 
@@ -96,7 +105,7 @@ To preload all files in the `src` directory and output to `result.txt`, while
 ignoring the `git` the `bin` directory, and the result file itself:
 
 ```bash
-./bin/code-preloader -o result.txt -i .git -i result.txt -i bin/ src
+./bin/code-preloader pack -o result.txt -i .git -i result.txt -i bin/ src
 ```
 
 #### Advanced Use Case
@@ -106,42 +115,136 @@ and appending prompts, while ignoring the `git` the `bin` directory, and the
 result file itself:
 
 ```bash
-./bin/code-preloader \
+./bin/code-preloader pack \
-    -o result.txt \
     -i .git -i bin/ -i result.txt -i prompts \
-    --header-prompt prompts/header-context.txt \
+    -H prompts/context.txt -F prompts/request-readme.txt \
-    --footer-prompt prompts/footer-write-readme.txt \
     src \
-    | xclip -selection clipboard -i
+    | ctrlc
 ```
 
-#### Advanced with configuration file
+__Note:__ the command `ctrlc` in previous command is an alias to `xclip
+-selection clipboard -i`
 
-You can also do the same by storing all parameters within a configuration file
+### Using a config file
-(ex: `code_preloader.yml`).
+
+You can automatically create an empty configuratio file by running
+Code-Preloader with the following command-line options:
 
 ```
+Usage: code-preloader init [options]
+
+Global options:
+    --version                        Show version
+    -h, --help                       Show this help
+
+Init options:
+    -c FILE, --config=FILE           Load parameters from FILE
+```
+
+#### Example: Advanced with configuration file
+
+Create an empty configuration file with
+
+```bash
+./bin/code-preloader init -c .code_preloader.yml
+```
+Adapt the configuration file (`.code_preloader.yml`) to your needs:
+
+```yaml
 ---
-
 ignore_list:
   - .git
-  - .code_preloader.yml
+  - code_preloader.yml
   - bin
   - prompts
 
-output_file_path: result.txt
+prompt:
+  header_path: prompts/context.txt
+  footer_path: prompts/request-readme.txt
+  template_path: null
 
-header_prompt_file_path: prompts/header-context.txt
+output_file_path: null
-
-footer_prompt_file_path: prompts/footer-write-readme.txt
 ```
 
-Then type
+Then you can type a shorter command like:
 
 ```bash
-./bin/code-preloader src | xclip -selection clipboard -i
+./bin/code-preloader pack -c .code_preloader.yml src | ctrlc
 ```
 
+__Note:__ the command `ctrlc` in previous command is an alias to `xclip
+-selection clipboard -i`
+
+### Prompting efficiently with CodePreloader
+
+The goal is to maximize the effectiveness of your interactions with large
+language models (LLMs) using CodePreloader for software development tasks.
+
+1. **Preload Your Project:** Before starting a new task, ensure you preload your project. This prepares the LLM with the necessary context, making it ready to assist you effectively.
+2. **Utilize Structure Separators:** Use "@@" as a delimiter to clearly separate and introduce new content. This could be for final requests, additional files, or other relevant data.
+3. **State Your Goals Clearly:** Begin by explicitly informing the LLM of your objectives. A clear and concise explanation of what you aim to achieve helps the model understand your needs better.
+4. **Reference Preloaded Files:** If necessary, refer to preloaded files by using "@@ File ...". This directs the LLM's attention to specific parts of your project.
+5. **Direct Focus Using Specific Phrasing:** Encourage the LLM to "focus" or "concentrate" on particular elements of the preloaded context, like files, classes, methods, etc. These specific terms have been observed to enhance the model's attention to the relevant details.
+6. **Approach the Task in Stages:** Divide your request into multiple stages for clarity and precision:
+   * **First Request**: Ask the LLM to analyze the content based on your instructions and explain the approach it would take for the changes requested. At this stage, request the LLM to refrain from writing code.
+   * **Second Request**: Instruct the LLM to focus on both the content and its previous analysis. Request a proposed fixed version of the specific file, class, or method based on the prior analysis. The more detailed your request, the better the outcome. Ask for changes to be made one at a time.
+   * **Third and Subsequent Requests**: Follow the same pattern as the second request but apply it to different parts of the project that require changes.
+
+#### Example: improving a README.md
+
+First request:
+
+> @@ REQUEST (ANALYZE)
+> 
+> We want to improve the README.md file as for an open-source project following the best practices.
+> 
+> Please focus on the code provided in the « @@ File ... » sections and analyze the existing « @@ File "README.md" », then take in account the content of class Config which analyzes the command line.
+> 
+> In this analysis of the existing « @@ File "README.md" », please provide me with a list of things that you appreciate, a list of things to amplify, a list of things to adjust.
+
+Second request:
+
+> @@ REQUEST (WRITE)
+> 
+> We want to improve the README.md file as for an open-source project following the best practices.
+> 
+> Please focus on the code provided in the « @@ File ... » sections and analyze the existing « @@ File "README.md" », then take in account the content of class Config which analyzes the command line.
+>
+> Please concentrate on the detail of your analysis in your previous message and write a new improved/fixed version of « @@ File "README.md"
+
+#### Example: writing a new feature
+
+First request:
+
+> @@ REQUEST (ANALYZE)
+>
+> I would like to change CLI parameters structure add add two subcommands:
+> * init : which will create an example .code_preloader.yml file (with comments)
+> * pack : which will create the packed version of the current directory for LLM prompting
+> 
+> Most of current options (except --version and --help) must become options of the pack subcommand.
+> 
+> I already started some changes to achieve this goal, but it is not finished, and I need your help and expert advises.
+> 
+> Can you please tell me :
+> * where the changes should occur (which file? which class? which method?)
+> * what kind of changes must be made there?
+> 
+> Please do not write code yet, simply explain.
+
+Second request:
+
+> @@ REQUEST (WRITE)
+>
+> I would like to change CLI parameters structure add add two subcommands:
+> * init : which will create an example .code_preloader.yml file (with comments)
+> * pack : which will create the packed version of the current directory for LLM prompting
+> 
+> Most of current options (except --version and --help) must become options of the pack subcommand.
+>
+> Please concentrate on the detail of your analysis in your previous message and write a new improved/fixed version of « @@ File "README.md". Please keep in mind the separation of concerns and the single responsibility principle.
+> 
+> Please provide me a new version of « File ... » which include the requested change?
 
 ## Contributing
 
@@ -151,8 +254,10 @@ appreciated**.
 
 ## Troubleshooting and Support
 
-If you encounter any issues or need support, please open an issue in the
+If you encounter any issues or need support, please open an issue in [the
-project's GitHub issue tracker. We strive to be responsive and helpful.
+project's issue
+tracker](https://code.apps.glenux.net/glenux/code-preloader/issues). We strive
+to be responsive and helpful.
 
 ## License
 

misc/config/dot.code_preloader.sample.yml

@@ -10,10 +10,10 @@ ignore_list:
   - prompts
   - Makefile
 
-output_file_path: null
+output_path: null
 
-header_prompt_file_path: null
+prompt:
-
+  header_path: null
-footer_prompt_file_path: prompts/footer.txt
+  footer_path: prompts/footer.txt
 
 #

misc/prompts/request-appreciative-analysis.txt

@@ -0,0 +1,7 @@
+Please focus on «@@ FILE ...» and analyze the content of «@@ FILE "{{FIXME: name of the file you want ot analyze}}"».
+
+Provide your analysis as an appreciative feedback, stating:
+- what you appreciate in current version
+- what you would amplify / augment
+- what you would adjust / change / remplace
+

misc/prompts/request-appreciative-analysis2.txt

@@ -0,0 +1,3 @@
+Focus on the existing «@@ FILE "README.md"». and concentrate on your analysis and appreciative feedback too.
+
+Please take in account each change you propose and write a fixed/improved version of the «@@ FILE "README.md"»

misc/prompts/request-crinja.txt

@@ -0,0 +1,10 @@
+I would like to replace the hardcoded structure of the output "@@ ... " with a jinja template that would be handler with the crinja.cr shard.
+
+My strategy is to preload the content during process_file and store it in a hash.
+Then the hash will be used in the end to render a final template (from crinja).
+
+Can you please tell me :
+* where the changes should occur (which file? which class? which method?)
+* what kind of changes must be made there?
+
+Please do not write code yet, simply explain.

misc/prompts/request-improve-readme1.txt

@@ -0,0 +1,7 @@
+
+We want to improve the README.md file as for an open-source project following the best practices.
+
+Please focus on the code provided in the « @@ File ... » sections and analyze the existing « @@ File "README.md" », then take in account the content of class Config which analyzes the command line.
+
+In this analysis of the existing « @@ File "README.md" », please provide me with a list of things that you appreciate, a list of things to amplify, a list of things to adjust.
+

misc/prompts/request-improve-readme2.txt

@@ -0,0 +1,7 @@
+
+We want to improve the README.md file as for an open-source project following the best practices.
+
+Please focus on the code provided in the « @@ File ... » sections and analyze the existing « @@ File "README.md" », then take in account the content of class Config which analyzes the command line.
+
+Please concentrate on the detail of your analysis in your previous message and write a new improved/fixed version of « @@ File "README.md"
+

misc/prompts/request-subcommand.txt

@@ -0,0 +1,13 @@
+I would like to change CLI parameters structure add add two subcommands:
+* init : which will create an example .code_preloader.yml file (with comments)
+* pack : which will create the packed version of the current directory for LLM prompting
+
+Most of current options (except --version and --help) must become options of the pack subcommand.
+
+I already started some changes to achieve this goal, but it is not finished, and I need your help and expert advises.
+
+Can you please tell me :
+* where the changes should occur (which File? which Class? and which method?)
+* what kind of changes must be made there?
+
+Please do not write code yet, simply explain.

misc/templates/default.j2

@@ -0,0 +1,16 @@
+{%- if prompt_header -%}
+@@ CONTEXT
+
+{{ prompt_header }}
+{%- endif -%}
+{%- for file in prompt_files -%}
+@@ FILE "{{ file.path }}" WITH MIME-TYPE "{{ file.mime_type }}"
+
+{{- file.content -}}
+
+{%- endfor -%}
+{%- if prompt_footer -%}
+@@ REQUEST
+
+{{ prompt_footer }}
+{%- endif -%}

prompts/footer-improve-readme.txt

@@ -1,10 +0,0 @@
-@@ REQUEST
-
-
-We want to improve the README.md file as for an open-source project following the best practices.
-
-Please focus on the code provided in the "@@ File..." sections and analyze the existing README.md.
-
-In this analysis, please provide me with a list of things that you appreciate, a list of things to amplify, a list of things to adjust.
-
-

shard.lock

@@ -1,5 +1,13 @@
 version: 2.0
 shards:
+  baked_file_system:
+    git: https://github.com/schovi/baked_file_system.git
+    version: 0.10.0
+
+  crinja:
+    git: https://github.com/straight-shoota/crinja.git
+    version: 0.8.1
+
   magic:
     git: https://github.com/dscottboggs/magic.cr.git
     version: 1.1.0

shard.yml

@@ -10,12 +10,19 @@ authors:
   - Glenn Y. Rolland <glenux@glenux.net>
 
 dependencies:
+  crinja:
+    github: straight-shoota/crinja
   magic:
     github: dscottboggs/magic.cr
   walk:
     github: alexherbo2/walk.cr
   version_from_shard:
     github: hugopl/version_from_shard
+  baked_file_system:
+    github: schovi/baked_file_system
+    version: 0.10.0
+  # completion:
+  #   github: f/completion
 
 # description: |
 #   Short description of chatgpt-preloader

spec/config_data/complex_config.yml

@@ -0,0 +1,22 @@
+---
+# Example configuration for Code-Preloader
+
+# List of repository paths to preload
+# repository_path_list:
+#   - "path/to/repo1"
+#   - "path/to/repo2"
+
+# List of patterns to ignore during preloading
+ignore_list:
+  - complex/ignore1
+  - complex/ignore2
+
+# Path to the output file (if null, output to STDOUT)
+output_path: complex_output.txt
+
+prompt:
+  # Optional: Path to a file containing the header prompt
+  header_path: null
+
+  # Optional: Path to a file containing the footer prompt
+  footer_path: null

spec/config_data/simple_config.yml

@@ -0,0 +1,21 @@
+---
+# Example configuration for Code-Preloader
+
+# List of repository paths to preload
+# repository_path_list:
+#   - "path/to/repo1"
+#   - "path/to/repo2"
+
+# List of patterns to ignore during preloading
+ignore_list:
+  - simple/ignore
+
+# Path to the output file (if null, output to STDOUT)
+output_path: simple_output.txt
+
+prompt:
+  # Optional: Path to a file containing the header prompt
+  header_path: null
+
+  # Optional: Path to a file containing the footer prompt
+  footer_path: null

spec/config_spec.cr

@@ -4,67 +4,109 @@ require "../src/config"
 CONFIG_FILE_SIMPLE = "spec/config_data/simple_config.yml"
 CONFIG_FILE_COMPLEX = "spec/config_data/complex_config.yml"
 
+alias Config = CodePreloader::Config
 describe CodePreloader::Config do
 
   context "Initialization" do
     it "initializes with default values" do
-      config = CodePreloader::Config.new
+      config = Config.new
-      config.repository_path_list.should eq [] of String
+      config.pack_options.should be_nil
-      config.ignore_list.should eq [] of String
+      config.init_options.should be_nil
-      config.output_file_path.should be_nil
-      config.header_prompt_file_path.should be_nil
-      config.footer_prompt_file_path.should be_nil
     end
   end
 
-  context "Parse Arguments" do
+  context "Handles global arguments" do
-    it "parses repository paths correctly" do
+    it "parses version option correctly" do
-      args = ["path/to/repo1", "path/to/repo2"]
+      args = ["--version"]
-      config = CodePreloader::Config.new
+      config = Config.new
       config.parse_arguments(args)
-      config.repository_path_list.should eq ["path/to/repo1", "path/to/repo2"]
+      config.subcommand == Config::Subcommand::Version
+      config.pack_options.should be_nil
+      config.init_options.should be_nil
+    end
+
+    it "parses help options correctly" do
+      args = ["-h"]
+      config = Config.new
+      config.parse_arguments(args)
+      config.subcommand.should eq(Config::Subcommand::Help)
+      config.pack_options.should be_nil
+      config.init_options.should be_nil
+
+      args = ["--help"]
+      config = Config.new
+      config.parse_arguments(args)
+      config.subcommand.should eq(Config::Subcommand::Help)
+      config.pack_options.should be_nil
+      config.init_options.should be_nil
+    end
+  end
+
+  context "Handles pack arguments" do
+    it "parses repository paths correctly" do
+      args = ["pack", "path/to/repo1", "path/to/repo2"]
+      config = Config.new
+      config.parse_arguments(args)
+      config.subcommand.should eq(Config::Subcommand::Pack)
+      config.pack_options.should be_truthy
+      config.pack_options.try do |opts| 
+        opts.source_list.should eq ["path/to/repo1", "path/to/repo2"]
+      end
     end
 
     it "parses ignore paths correctly" do
-      args = ["-i", "path/to/ignore", "path/to/repo"]
+      args = ["pack", "-i", "path/to/ignore", "path/to/repo"]
-      config = CodePreloader::Config.new
+      config = Config.new
       config.parse_arguments(args)
-      config.ignore_list.should eq ["path/to/ignore"]
+      config.subcommand.should eq(Config::Subcommand::Pack)
+      config.pack_options.should be_truthy
+      config.pack_options.try do |opts|
+        opts.ignore_list.should eq ["path/to/ignore"]
+      end
     end
 
     it "parses output file path correctly" do
-      args = ["-o", "output.txt", "path/to/repo"]
+      args = ["pack", "-o", "output.txt", "path/to/repo"]
-      config = CodePreloader::Config.new
+      config = Config.new
       config.parse_arguments(args)
-      config.output_file_path.should eq "output.txt"
+      config.subcommand.should eq(Config::Subcommand::Pack)
+      config.pack_options.should be_truthy
+      config.pack_options.try do |opts|
+        opts.output_path.should eq "output.txt"
+      end
     end
 
   end
 
 
-  context "Config File Loading" do
+  context "loads config file" do
     it "loads settings from a simple config file" do
-      config = CodePreloader::Config.new
+      config = Config.new
-      args = ["-c", CONFIG_FILE_SIMPLE, "path/to/repo"]
+      args = ["pack", "-c", CONFIG_FILE_SIMPLE, "path/to/repo"]
       config.parse_arguments(args)
       
       # Assuming the simple_config.yml has specific settings
-      config.repository_path_list.should eq ["simple/repo/path"]
+      config.pack_options.should be_truthy
-      config.ignore_list.should eq ["simple/ignore"]
+      config.pack_options.try do |opts|
-      config.output_file_path.should eq "simple_output.txt"
+        opts.source_list.should eq ["path/to/repo"]
-      # ... assertions for other properties if needed ...
+        opts.ignore_list.should eq ["simple/ignore"]
+        opts.output_path.should eq "simple_output.txt"
+      end
     end
 
     it "loads settings from a complex config file" do
       repo_path ="path/to/repo"
-      config = CodePreloader::Config.new
+      config = Config.new
-      args = ["-c", CONFIG_FILE_COMPLEX, repo_path]
+      args = ["pack", "-c", CONFIG_FILE_COMPLEX, repo_path]
       config.parse_arguments(args)
       
       # Assuming the complex_config.yml has specific settings
-      config.repository_path_list.should eq [repo_path]
+      config.pack_options.should be_truthy
-      config.ignore_list.should eq ["complex/ignore1", "complex/ignore2"]
+      config.pack_options.try do |opts|
-      config.output_file_path.should eq "complex_output.txt"
+        opts.source_list.should eq [repo_path]
+        opts.ignore_list.should eq ["complex/ignore1", "complex/ignore2"]
+        opts.output_path.should eq "complex_output.txt"
+      end
     end
 
   end

spec/spec_helper.cr

@@ -1,4 +1,5 @@
 
 require "spec"
 
-# require "../src/"
+ENV["CODE_PRELOADER_DETECT"] = "no"
+

src/cli.cr

@@ -1,85 +1,174 @@
-
+require "colorize"
-# vim: set ts=2 sw=2 et ft=crystal:
-
 require "file"
 require "option_parser"
 require "magic"
+require "crinja"
 
 require "./config"
 require "./filelist"
+require "./file_storage"
 
 # The CodePreloader module organizes classes and methods related to preloading code files.
 module CodePreloader
   # The Cli class handles command-line interface operations for the CodePreloader.
+
   class Cli
+    alias ProcessedFile = NamedTuple(path: String, content: String, mime_type: String)
+
     @config : Config
 
     # Initializes the Cli class with default values.
     def initialize(args)
-      @output_file_path = ""
+      @output_path = ""
       @config = Config.new()
       @config.parse_arguments(args)
     end
 
       # Executes the main functionality of the CLI application.
     def exec
-      # get local values for typing
+      case @config.subcommand
-      output_file_path = @output_file_path
+      when Config::Subcommand::Init then exec_init(@config.init_options)
-      repository_path_list = @config.repository_path_list
+      when Config::Subcommand::Pack then exec_pack(@config.pack_options)
-      header_prompt_file_path = @config.header_prompt_file_path
+      when Config::Subcommand::Version then exec_version
-      footer_prompt_file_path = @config.footer_prompt_file_path
+      when Config::Subcommand::Help then exec_help
+      when Config::Subcommand::None then exec_none
+      else
+        abort("Unknown subcommand #{@config.subcommand}!")
+      end
+    end
+
+    def exec_init(init_options)
+      abort("Unexpected nil value for init_options!") if init_options.nil?
+
+      # Default path for the .code_preloader.yml file
+      default_config_path = ".code_preloader.yml"
+
+      # Use the specified path if provided, otherwise use the default
+      config_path = init_options.config_path || default_config_path
+      abort("ERROR: configuration file already exist: #{config_path}") if File.exists? config_path
+
+      # Content of the default .code_preloader.yml file 
+      config_content = FileStorage.get("default_config.yml").gets_to_end
+
+      # Writing the configuration content to the file
+      File.write(config_path, config_content)
+      puts "Configuration file created at: #{config_path}"
+    rescue e : Exception
+      abort("ERROR: Unable to create the configuration file: #{e.message}")
+    end
+    
+    def exec_version
+      puts "#{PROGRAM_NAME} v#{VERSION}"
+      exit(0)
+    end
+
+    def exec_none
+      STDERR.puts @config.parser
+      abort("ERROR: No command specified!")
+    end
+
+    def exec_help
+      @config.help_options.try do |opts|
+        puts opts.parser_snapshot
+      end
+      exit(0)
+    end
+
+    def exec_pack(pack_options)
+      abort("Unexpected nil value for pack_options!") if pack_options.nil?
+
+      preloaded_content = {} of String => NamedTuple(mime: String, content: String)
+      config_path = pack_options.config_path
+      output_path = pack_options.output_path
+      source_list = pack_options.source_list
+      prompt_header_path = pack_options.prompt_header_path
+      prompt_footer_path = pack_options.prompt_footer_path
+      prompt_template_path = pack_options.prompt_template_path
+      regular_output_file = false
+      prompt_header_content = nil
+      prompt_footer_content = nil
+      prompt_template_content = ""
+      STDERR.puts "Loading config file from: #{config_path}".colorize(:yellow)
 
       filelist = FileList.new()
-      filelist.add(repository_path_list)
+      filelist.add(source_list)
-      @config.ignore_list.each do |ignore_pattern|
+      pack_options.ignore_list.each do |ignore_pattern|
         filelist.reject { |path| !!(path =~ Regex.new(ignore_pattern)) }
       end
 
-      if !header_prompt_file_path.nil?
+      STDERR.puts "Loading template file from: #{prompt_template_path ? prompt_template_path : "<internal>" }".colorize(:yellow)
-        STDERR.puts "Loading header prompt from: #{header_prompt_file_path}"
+      if prompt_template_path 
-        header_prompt = File.read(header_prompt_file_path)
+        prompt_template_content = File.read(prompt_template_path)
+      else
+        prompt_template_content = FileStorage.get("default_template.j2").gets_to_end
       end
 
-      if !footer_prompt_file_path.nil?
+
-        STDERR.puts "Loading footer prompt from: #{footer_prompt_file_path}"
+      if !prompt_header_path.nil?
-        footer_prompt = File.read(footer_prompt_file_path)
+        STDERR.puts "Loading header prompt from: #{prompt_header_path}".colorize(:yellow)
+        prompt_header_content = File.read(prompt_header_path)
       end
 
-      unless output_file_path.nil? || output_file_path.try(&.empty?) || (output_file_path != "-")
+      if !prompt_footer_path.nil?
-        output_file = File.open(output_file_path, "w")
+        STDERR.puts "Loading footer prompt from: #{prompt_footer_path}".colorize(:yellow)
-        invalid_output_file = false
+        prompt_footer_content = File.read(prompt_footer_path)
       end
 
-      invalid_output_file = true
       output_file = STDOUT
-      header_prompt = ""
+      output_path.try do |path|
-      footer_prompt = ""
+        break if path.empty?
+        break if path == "-"
+        regular_output_file = true
+        output_file = File.open(path, "w")
+      end
+      STDERR.puts "Writing output to: #{regular_output_file ? output_path : "stdout" }".colorize(:yellow)
 
-      output_file.puts header_prompt if @config.header_prompt_file_path
+      # FIXME: prompt_header_path.try { output_file.puts prompt_header_content }
 
-      STDERR.puts "Processing repository: #{@config.repository_path_list}"
+      STDERR.puts "Processing source directories: #{source_list}".colorize(:yellow)
-      filelist.each do |file_path|
+      processed_files = [] of ProcessedFile
-        process_file(file_path, output_file)
+      filelist.to_a.sort.each do |file_path|
+        STDERR.puts "Processing file: #{file_path}".colorize(:yellow)
+        file_result = process_file(file_path, output_file)
+        processed_files << file_result
       end
 
-      output_file.puts footer_prompt if @config.footer_prompt_file_path
+      # FIXME: prompt_footer_path.try { output_file.puts prompt_footer_content }
 
-      output_file.close if !invalid_output_file
+      output_file.puts Crinja.render(
-      STDERR.puts "Processing completed. Output written to: #{invalid_output_file ? "stdout" : output_file_path}"
+        prompt_template_content, 
+        { 
+          "prompt_header": prompt_header_content,
+          "prompt_files": processed_files,
+          "prompt_footer": prompt_footer_content
+        }
+      ) 
+
+      output_file.close if regular_output_file
+      STDERR.puts "Processing completed.".colorize(:yellow)
 
     rescue e : Exception
-      STDERR.puts "An error occurred during execution: #{e.message}"
+      STDERR.puts "ERROR: #{e.message}"
       exit(1)
     end
 
     private def process_file(file_path : String, output_file : IO::FileDescriptor)
-      fh = File.open(file_path)
+      mime = ""
-      mime = Magic.mime_type.of(fh)
+      clean_content = ""
-      output_file.puts "@@ File \"#{file_path}\" (Mime-Type: #{mime.inspect})"
+      File.open(file_path) do |fh|
-      output_file.puts ""
+        mime = Magic.mime_type.of(fh)
-      output_file.puts(fh.gets_to_end)
+        clean_content = (
-      output_file.puts ""
+          fh.gets_to_end
-      fh.close
+          .strip
+          .gsub(/\n\s*\n\s*\n/,"\n\n")
+        )
+      end
+
+      return {
+        path: file_path,
+        content: clean_content,
+        mime_type: mime
+      }
     end
   end
 end

src/config.cr

@@ -1,108 +1,297 @@
 require "option_parser"
 require "yaml"
+# require "completion"
 
 require "./models/root_config"
 require "./version"
 
 module CodePreloader
   class Config
-    property repository_path_list : Array(String) = [] of String
+
-    property ignore_list : Array(String) = [] of String
+    enum Subcommand
-    property output_file_path : String?
+      None
-    property header_prompt_file_path : String?
+      Init
-    property footer_prompt_file_path : String?
+      Pack
+      Help
+      Version
+    end
+
+    class HelpOptions
+      property parser_snapshot : OptionParser? = nil
+    end
+
+    class InitOptions
+      property config_path : String? = nil
+    end
+
+    class PackOptions
+      property config_path : String? = nil
+      property source_list : Array(String) = [] of String
+      property ignore_list : Array(String) = [] of String
+      property output_path : String?
+      property prompt_template_path : String?
+      property prompt_header_path : String?
+      property prompt_footer_path : String?
+    end
+
+    getter verbose : Bool = false
+    getter parser : OptionParser?
+    getter subcommand : Subcommand = Subcommand::None
+    getter pack_options : PackOptions?
+    getter init_options : InitOptions?
+    getter help_options : HelpOptions?
 
     def initialize()
     end
 
-    def parse_arguments(args : Array(String))
+    def parse_init_options(parser)
-      OptionParser.parse(args) do |parser|
+      @init_options = InitOptions.new
-        parser.banner = "Usage: code-preloader [options] DIR1 ..."
 
-        parser.on(
+      parser.banner = [
-          "-c CONFIG_FILE", 
+        "Usage: code-preloader init [options]\n",
-          "--config=CONFIG_FILE", 
+        "Global options:"
-          "Load parameters from CONFIG_FILE"
+      ].join("\n")
-        ) do |config_file|
-          load_config(config_file)
-        end
 
-        parser.on(
+      parser.separator "\nInit options:"
-          "-i IGNORE_PATH", 
+      parser.unknown_args do |remaining_args, _|
-          "--ignore=IGNORE_PATH", 
+        # FIXME: detect and make error if there are more or less than one
-          "Ignore file or directory"
+        remaining_args.each do |arg|
-        ) do |ignore_file|
+          @init_options.try &.config_path = arg
-          @ignore_list << ignore_file
-        end
-
-        parser.on(
-          "-o OUTPUT_FILE", 
-          "--output=OUTPUT_FILE", 
-          "Write output to OUTPUT_FILE"
-        ) do |output_file|
-          @output_file_path = output_file
-        end
-
-        parser.on(
-          "-H HEADER_PROMPT_FILE", 
-          "--header-prompt=HEADER_PROMPT_FILE", 
-          "Load header prompt from HEADER_PROMPT_FILE"
-        ) do |header_prompt_file|
-          @header_prompt_file_path = header_prompt_file
-        end
-
-        parser.on(
-          "-F FOOTER_PROMPT_FILE", 
-          "--footer-prompt=FOOTER_PROMPT_FILE", 
-          "Load footer prompt from FOOTER_PROMPT_FILE"
-        ) do |footer_prompt_file|
-          @footer_prompt_file_path = footer_prompt_file
-        end
-
-        parser.on("--version", "Show version") do
-          STDOUT.puts "#{PROGRAM_NAME} #{VERSION}"
-          exit(0)
-        end
-
-        parser.on("-h", "--help", "Show this help") do
-          STDERR.puts parser
-          exit
-        end
-
-        parser.unknown_args do |remaining_args, _|
-          remaining_args.each do |arg|
-            @repository_path_list << arg
-          end
         end
       end
 
+      parser.on(
+        "-c FILE", 
+        "--config=FILE", 
+        "Load parameters from FILE"
+      ) do |config_file|
+        @init_options.try { |opt| opt.config_path = config_file }
+      end
+
+      parser.separator ""
+
+      parser.missing_option do |opt|
+        puts parser
+        abort("ERROR: Missing parameter for option #{opt}!")
+      end
+
+      parser.invalid_option do |opt|
+        puts parser
+        abort("ERROR: Invalid option #{opt}!")
+      end
+
+      # complete_with "code-preloader init", parser
+    end
+
+    def parse_pack_options(parser) 
+      @pack_options = PackOptions.new
+
+      unless ENV["CODE_PRELOADER_DETECT"]? =~ /(no|false|0)/i
+        config_file = detect_config_file 
+        config_file.try { |path| load_pack_config(path) }
+      end
+
+      parser.banner = [
+        "Usage: code-preloader pack [options] DIR ...\n",
+        "Global options:"
+      ].join("\n")
+
+      parser.separator "\nPack options:"
+
+      parser.on(
+        "-c FILE", 
+        "--config=FILE", 
+        "Load parameters from FILE\n(default: autodetect)"
+      ) do |config_file|
+        @pack_options.try { |opt| load_pack_config(config_file) }
+      end
+
+      parser.on(
+        "-F FILE", 
+        "--prompt-footer=FILE", 
+        "Load prompt footer from FILE (default: none)"
+      ) do |prompt_footer_path|
+        @pack_options.try { |opt| opt.prompt_footer_path = prompt_footer_path }
+      end
+
+      parser.on(
+        "-H FILE", 
+        "--prompt-header=FILE", 
+        "Load prompt header from FILE (default: none)"
+      ) do |prompt_header_path|
+        @pack_options.try { |opt| opt.prompt_header_path = prompt_header_path }
+      end
+
+      parser.on(
+        "-i REGEXP", 
+        "--ignore=REGEXP", 
+        "Ignore file or directory. Can be used\nmultiple times (default: none)"
+      ) do |ignore_file|
+        @pack_options.try { |opt| opt.ignore_list << ignore_file }
+      end
+
+      parser.on(
+        "-o FILE", 
+        "--output=FILE", 
+        "Write output to FILE (default: \"-\", STDOUT)"
+      ) do |output_file|
+        @pack_options.try { |opt| opt.output_path = output_file }
+      end
+
+      parser.on(
+        "-t FILE", 
+        "--template=FILE", 
+        "Load template from FILE (default: internal)"
+      ) do |prompt_template_path|
+        @pack_options.try { |opt| opt.prompt_template_path = prompt_template_path }
+      end
+
+      parser.separator ""
+
+      parser.unknown_args do |remaining_args, _|
+        remaining_args.each do |arg|
+          @pack_options.try { |opt| opt.source_list << arg }
+        end
+      end
+
+      parser.missing_option do |opt|
+        puts parser
+        abort("ERROR: Missing parameter for option #{opt}!")
+      end
+
+      parser.invalid_option do |ex|
+        puts parser
+        abort("ERROR: Invalid option #{ex}")
+      end
+
+      # complete_with "code-preloader pack", parser
+    end
+
+    def parse_arguments(args : Array(String))
+      @parser = OptionParser.new do |parser|
+        parser.banner = [
+          "Usage: code-preloader <subcommand> [options] [DIR] [...]\n",
+          "Global options:"
+        ].join("\n")
+
+        parser.on("-h", "--help", "Show this help") do
+          @subcommand = Subcommand::Help
+          @help_options = HelpOptions.new
+          @help_options.try do |opts|
+            opts.parser_snapshot = parser.dup
+          end
+        end
+
+        parser.on("-v", "--verbose", "Enable verbose mode") do
+          @verbose = true
+        end
+
+        parser.on("--version", "Show version") do
+          @subcommand = Subcommand::Version
+        end
+
+
+        parser.separator "\nSubcommands:"
+
+        parser.on("init", "Create an example .code_preloader.yml file") do
+          @subcommand = Subcommand::Init
+          parse_init_options(parser)
+        end
+
+        parser.on("pack", "Create the packed version of a directory for LLM prompting") do
+          @subcommand = Subcommand::Pack
+          parse_pack_options(parser)
+        end
+
+        parser.separator ""
+
+        parser.invalid_option do |ex|
+          puts parser
+          abort("ERROR: Invalid option #{ex}")
+        end
+
+        # complete_with "code-preloader", parser
+      end
+
+      @parser.try &.parse(args)
       validate
     end
 
-    private def validate
+    def detect_config_file() : String?
-      abort("Missing repository path.") if @repository_path_list.empty?
+      home_dir = ENV["HOME"]
+      possible_files = [
+        File.join(".code_preloader.yaml"),
+        File.join(".code_preloader.yml"),
+        File.join(home_dir, ".config", "code_preloader", "config.yaml"),
+        File.join(home_dir, ".config", "code_preloader", "config.yml"),
+        File.join(home_dir, ".config", "code_preloader.yaml"),
+        File.join(home_dir, ".config", "code_preloader.yml"),
+        File.join("/etc", "code_preloader", "config.yaml"),
+        File.join("/etc", "code_preloader", "config.yml"),
+      ]
 
-      STDERR.puts("Output file path not specified (using STDOUT)") if @output_file_path.nil? || @output_file_path.try(&.empty?)
+      possible_files.each do |file_path|
+        return file_path if File.exists?(file_path)
+      end
+
+      return nil
+    end
+
+    private def validate
+      case @subcommand
+      when Subcommand::Init then validate_init
+      when Subcommand::Pack then validate_pack
+      when Subcommand::None, Subcommand::Help, Subcommand::Version
+        # do nothing
+      else
+        abort("Unknown subcommand #{@subcommand}")
+      end
+    end
+
+    private def validate_init
+      abort("No init options defined!") if @init_options.nil?
+    end
+
+    private def validate_pack
+      opts = @pack_options
+      abort("No pack options defined!") if opts.nil?
+      abort("Missing repository path.") if opts.source_list.empty?
     end
 
     # Reads and returns a list of paths to ignore from the given file.
-    def self.get_ignore_list(ignore_file_path : String) : Array(String)
+    def self.get_ignore_list(ignore_path : String) : Array(String)
-      File.exists?(ignore_file_path) ? File.read_lines(ignore_file_path).map(&.strip) : [] of String
+      File.exists?(ignore_path) ? File.read_lines(ignore_path).map(&.strip) : [] of String
     rescue e : IO::Error
       STDERR.puts "Error reading ignore file: #{e.message}"
       exit(1)
     end
 
-    private def load_config(config_file_path : String)
+    private def load_pack_config(config_path : String)
-      config_str = File.read(config_file_path)
+      opts = @pack_options
+      abort("No pack options defined!") if opts.nil?
 
+      config_str = File.read(config_path)
       root = Models::RootConfig.from_yaml(config_str)
 
-      @repository_path = root.repository_path_list || @repository_path_list
+      opts.config_path = config_path
-      @ignore_list = root.ignore_list || @ignore_list
+      if opts.source_list.nil? || opts.source_list.try &.empty?
-      @output_file_path = root.output_file_path || @output_file_path
+        root.source_list.try { |value| opts.source_list = value }
-      @header_prompt_file_path = root.header_prompt_file_path || @header_prompt_file_path
+      end
-      @footer_prompt_file_path = root.footer_prompt_file_path || @footer_prompt_file_path
+      if opts.ignore_list.nil? || opts.ignore_list.try &.empty?
+        root.ignore_list.try { |value| opts.ignore_list = value }
+      end
+      if opts.output_path.nil?
+        opts.output_path = root.output_path 
+      end
+      if opts.prompt_header_path.nil?
+        root.prompt.try &.header_path.try { |value| opts.prompt_header_path = value }
+      end
+      if opts.prompt_footer_path.nil?
+        root.prompt.try &.footer_path.try { |value| opts.prompt_footer_path = value }
+      end
+      if opts.prompt_template_path.nil?
+        root.prompt.try &.template_path.try { |value| opts.prompt_template_path = value }
+      end
 
     rescue ex : Exception
       STDERR.puts "Failed to load config file: #{ex.message}"

src/file_storage.cr

@@ -0,0 +1,7 @@
+require "baked_file_system"
+
+class FileStorage
+  extend BakedFileSystem
+
+  bake_folder "../static"
+end

src/filelist.cr

@@ -2,10 +2,8 @@
 require "walk"
 
 module CodePreloader
-
   # Manage a list of files
   class FileList 
-
     alias Filter = String -> Bool
 
     class NotADirectory < Exception
@@ -48,7 +46,7 @@ module CodePreloader
       seen = Set(String).new
 
       # walk each source
-      @sources.each do |dir|
+      @sources.sort.each do |dir|
         walker = Walk::Down.new(dir)
 
         walker = walker.filter do |path|
@@ -56,15 +54,16 @@ module CodePreloader
           keep = true
           must_select = false
           must_reject = false
+          clean_path = path.to_s.gsub(/^\.\//,"")
 
           @filters_in.each do |filter_in|
-            must_select = must_select || filter_in.call(path.to_s)
+            must_select = must_select || filter_in.call(clean_path)
           end
           keep = keep && must_select if @filters_in.any?
           keep = keep || is_dir
 
           @filters_out.each do |filter_out|
-            must_reject = must_reject || filter_out.call(path.to_s)
+            must_reject = must_reject || filter_out.call(clean_path)
           end
           keep = keep && !must_reject if @filters_out.any?
 
@@ -72,13 +71,14 @@ module CodePreloader
         end
 
         walker.each do |path|
-          next if File.directory? path
+          clean_path = path.to_s.gsub(/^\.\//,"")
+          next if File.directory? clean_path
 
-          path = File.realpath(path) if File.symlink? path
+          path = File.realpath(path) if File.symlink? clean_path
-          next if seen.includes? path.to_s
+          next if seen.includes? clean_path
 
-          seen << path.to_s
+          seen << clean_path
-          yield path.to_s
+          yield clean_path
         end
       end
     end
@@ -88,7 +88,7 @@ module CodePreloader
       self.each do |path|
         files << path.to_s
       end
-      files
+      files.sort
     end
   end
 end

src/main.cr

@@ -1,10 +1,5 @@
-
-# vim: set ts=2 sw=2 et ft=crystal:
-
 require "./cli"
 
-
-# Now that we have checked for nil, it's safe to use not_nil!
 app = CodePreloader::Cli.new(ARGV)
 app.exec()
 

src/models/prompt_config.cr

@@ -0,0 +1,18 @@
+
+require "yaml"
+
+module CodePreloader::Models
+  class PromptConfig
+    include YAML::Serializable
+    include YAML::Serializable::Strict
+
+    @[YAML::Field(key: "header_path")]
+    getter header_path : String?
+
+    @[YAML::Field(key: "footer_path")]
+    getter footer_path : String?
+
+    @[YAML::Field(key: "template_path")]
+    getter template_path : String?
+  end
+end

src/models/root_config.cr

@@ -1,22 +1,20 @@
 
 require "yaml"
+require "./prompt_config"
 
 module CodePreloader::Models
   class RootConfig
     include YAML::Serializable
     include YAML::Serializable::Strict
 
-    @[YAML::Field(key: "repository_path_list")]
+    @[YAML::Field(key: "source_list")]
-    getter repository_path_list : Array(String)?
+    getter source_list : Array(String)?
 
-    @[YAML::Field(key: "output_file_path")]
+    @[YAML::Field(key: "output_path")]
-    getter output_file_path : String?
+    getter output_path : String?
 
-    @[YAML::Field(key: "header_prompt_file_path")]
+    @[YAML::Field(key: "prompt")]
-    getter header_prompt_file_path : String?
+    getter prompt : PromptConfig?
-
-    @[YAML::Field(key: "footer_prompt_file_path")]
-    getter footer_prompt_file_path : String?
 
     @[YAML::Field(key: "ignore_list")]
     getter ignore_list : Array(String)?

static/default_config.yml

@@ -0,0 +1,24 @@
+---
+# Example configuration for Code-Preloader
+
+# List of repository paths to preload
+# source_list:
+#   - "path/to/repo1"
+#   - "path/to/repo2"
+
+# List of patterns to ignore during preloading
+ignore_list:
+  - ^\.git/.*
+
+# Path to the output file (if null, output to STDOUT)
+output_path: null
+
+prompt:
+  # Optional: Path to a file containing the prompt header
+  header_path: null
+
+  # Optional: Path to a file containing the prompt footer
+  footer_path: null
+
+  # Optional: Path to a file container a jinja template to structure the prompt
+  template_path: null

static/default_template.j2

@@ -0,0 +1,16 @@
+{%- if prompt_header -%}
+@@ CONTEXT
+
+{{ prompt_header }}
+{%- endif -%}
+{%- for file in prompt_files -%}
+@@ FILE "{{ file.path }}" WITH MIME-TYPE "{{ file.mime_type }}"
+
+{{- file.content -}}
+
+{%- endfor -%}
+{%- if prompt_footer -%}
+@@ REQUEST
+
+{{ prompt_footer }}
+{%- endif -%}