Merge tag 'add-documentation-for-config' into develop

v0.1

TODO.md

@@ -0,0 +1,9 @@
+cat database.sql.gz |ssh shiva-ratri.infra.boldcode.io 'docker exec -i customer-sans-a-site.web.1 sh -c "cat > adminer.sql.gz"'
+
+cat database.sql |ssh shiva-ratri.infra.boldcode.io 'dokku mariadb:import customer-sans-a-wpsandbox'
+
+# TODO
+
+* verify target app exist
+* verify target database exist
+

doc/pushokku-v3.yml

@@ -0,0 +1,61 @@
+---
+version: "3"
+
+hosts:
+  - name: local
+    localhost: {}
+
+  - name: testing
+    ssh:
+      user: debian
+      host: shiva-ratri.infra.boldcode.io
+
+
+endpoints:
+  # Local endpoints
+  - name: local-wp-image
+    host: local
+    docker_image:
+      name: sans-a-site-v2-wordpress_wordpress
+
+  - name: local-db-dump
+    host: local
+    mysql_dump:
+      path: database.sql
+
+  - name: local-db-script
+    host: local
+    script:
+      path: .pushokku/post-update.sh
+
+  # Remote endpoints
+  - name: remote-wp-app
+    host: testing
+    dokku_app:
+      name: customer-sans-a-site
+
+  - name: remote-wp-db
+    host: testing
+    dokku_mariadb:
+      name: customer-sans-a-wpsandbox
+
+
+filters:
+  - name: compress
+    dual:
+      cmd_in: gzip -
+      cmd_out: gunzip -
+
+
+deployments:
+  - transfer:
+      src: local-wp-image
+      dest: remote-wp-app
+
+  - transfer:
+      src: local-db-dump
+      dest: remote-wp-db
+
+  - run:
+      src: local-db-script
+      dest: remote-wp-app

src/config.cr

@@ -1,9 +1,52 @@
 
 require "yaml"
-require "./config/config"
+require "./config/*"
-require "./config/local"
-require "./config/remote"
-require "./config/deployment"
 
+class Config
+  YAML.mapping(
+    version: String,
+    hosts: Array(HostConfig),
+    endpoints: Array(EndpointConfig),
+    filters: Array(FilterConfig),
+    deployments: Array(DeploymentConfig)
+  )
 
+  alias FindableConfig = 
+    Array(HostConfig) | 
+    Array(EndpointConfig) | 
+    Array(FilterConfig) | 
+    Array(DeploymentConfig)
 
+  class MissingItemError < Exception 
+  end
+
+  class MultipleItemsError < Exception
+  end
+
+  def find(list : FindableConfig, name : String, str : String)
+    matches = list.select { |item| item.name == name }
+    if matches.size > 1 
+      raise MultipleItemsError.new "Multiple #{str} have the name #{str}"
+    end
+    if matches.size < 1
+      raise MissingItemError.new "No #{str} found for name #{name}"
+    end
+    return matches.first
+  end
+
+  def find_host(name : String) : HostConfig
+    find(self.hosts, name, "hosts")
+  end
+
+  def find_endpoint(name : String) : EndpointConfig
+    find(self.endpoints, name, "endpoints")
+  end
+
+  def find_filter(name : String) : FilterConfig
+    find(self.filters, name, "filters")
+  end
+
+  def find_deployment(name : String) : DeploymentConfig
+    find(self.deployments, name, "deployments")
+  end
+end

src/config/config.cr

@@ -1,14 +0,0 @@
-
-require "yaml"
-require "./local"
-require "./remote"
-require "./deployment"
-
-class Config
-  YAML.mapping(
-    version: String,
-    locals: Array(LocalConfig),
-    remotes: Array(RemoteConfig),
-    deployments: Array(DeploymentConfig)
-  )
-end

src/config/deployment.cr

@@ -1,37 +1,21 @@
 
 require "yaml"
+require "./deployment_settings"
 
-class DokkuMariadbConfig
+class RunDeploymentConfig
   YAML.mapping(
-    name: String,
+    name: String?,
-    options: YAML::Any | Nil
+    run: RunDeploymentConfigSettings
   )
 end
 
-class DokkuAppConfig
+class TransferDeploymentConfig
   YAML.mapping(
-    name: String,
+    name: String?,
-    options: YAML::Any | Nil
+    transfer: TransferDeploymentConfigSettings
-  )
-end
-
-class DeploymentMariadbConfig
-  YAML.mapping(
-    local: String,
-    remote: String,
-    dokku_mariadb: DokkuMariadbConfig,
-  )
-end
-
-class DeploymentAppConfig
-  YAML.mapping(
-    local: String,
-    remote: String,
-    dokku_app: DokkuAppConfig,
   )
 end
 
 alias DeploymentConfig = 
-  DeploymentMariadbConfig | 
+  TransferDeploymentConfig |
-  DeploymentAppConfig
+  RunDeploymentConfig
-

src/config/deployment_settings.cr

@@ -0,0 +1,17 @@
+
+require "yaml"
+
+class TransferDeploymentConfigSettings
+  YAML.mapping(
+    src: String,
+    dest: String,
+    filters: Array(String)?
+  )
+end
+
+class RunDeploymentConfigSettings
+  YAML.mapping(
+    src: String,
+    dest: String,
+  )
+end

src/config/endpoint.cr

@@ -0,0 +1,52 @@
+
+require "yaml"
+require "./endpoint_settings"
+
+
+class DokkuMariadbEndpointConfig
+  YAML.mapping(
+    name: String,
+    host: String,
+    dokku_mariadb: DokkuMariadbEndpointConfigSettings
+  )
+end
+
+class DokkuAppEndpointConfig
+  YAML.mapping(
+    name: String,
+    host: String,
+    dokku_app: DokkuAppEndpointConfigSettings
+  )
+end
+
+class ScriptEndpointConfig
+  YAML.mapping(
+    name: String,
+    host: String,
+    script: ScriptEndpointConfigSettings
+  )
+end
+
+class MysqlDumpEndpointConfig
+  YAML.mapping(
+    name: String,
+    host: String,
+    mysql_dump: MysqlDumpEndpointConfigSettings
+  )
+end
+
+class DockerImageEndpointConfig
+  YAML.mapping(
+    name: String,
+    host: String,
+    docker_image: DockerImageEndpointConfigSettings
+  )
+end
+
+alias EndpointConfig =
+  DockerImageEndpointConfig |
+  MysqlDumpEndpointConfig |
+  ScriptEndpointConfig |
+  DokkuAppEndpointConfig |
+  DokkuMariadbEndpointConfig
+

src/config/endpoint_settings.cr

@@ -0,0 +1,45 @@
+
+require "yaml"
+
+class DokkuMariadbEndpointConfigSettings
+  YAML.mapping(
+    name: String
+  )
+end
+
+class DokkuAppEndpointConfigSettings
+  YAML.mapping(
+    name: String
+  )
+end
+
+class DockerImageEndpointConfigSettings
+  YAML.mapping(
+    name: String,
+    tag: { 
+      type: String,
+      nilable: false,
+      default: "latest"
+    }
+  )
+
+  def initialize(@name : String, @tag : String)
+  end
+
+  def to_s 
+    "#{name}:#{tag}"
+  end
+end
+
+class MysqlDumpEndpointConfigSettings
+  YAML.mapping(
+    path: String
+  )
+end
+
+class ScriptEndpointConfigSettings
+  YAML.mapping(
+    path: String
+  )
+end
+

src/config/filter.cr

@@ -0,0 +1,22 @@
+
+require "yaml"
+require "./filter_settings"
+
+
+class DualFilterConfig
+  YAML.mapping(
+    name: String,
+    dual: DualFilterConfigSettings
+  )
+end
+
+class MonoFilterConfig
+  YAML.mapping(
+    name: String,
+    cmd: String
+  )
+end
+
+alias FilterConfig = 
+  MonoFilterConfig |
+  DualFilterConfig

src/config/filter_settings.cr

@@ -0,0 +1,9 @@
+
+require "yaml"
+
+class DualFilterConfigSettings
+  YAML.mapping(
+    cmd_in: String,
+    cmd_out: String,
+  )
+end

src/config/host.cr

@@ -0,0 +1,29 @@
+
+require "yaml"
+require "./host_settings"
+
+class LocalHostConfig
+  YAML.mapping(
+    name: String,
+    localhost: Hash(String, YAML::Any)
+  )
+
+  def to_s
+    "LocalHost[#{name}]"
+  end
+end
+
+
+class SshHostConfig
+  YAML.mapping(
+    name: String,
+    ssh: SshHostConfigSettings
+  )
+  def to_s
+    "SshHost[#{name}]"
+  end
+end
+
+alias HostConfig = 
+  LocalHostConfig |
+  SshHostConfig

src/config/host_settings.cr

@@ -1,11 +1,13 @@
 
 require "yaml"
 
-class RemoteConfig
+class SshHostConfigSettings
   YAML.mapping(
-    name: String,
     user: String,
     host: String
   )
-end
 
+  def to_s 
+    "#{user}@#{host}"
+  end
+end

src/config/local.cr

@@ -1,32 +0,0 @@
-
-require "yaml"
-
-enum LocalType
-  DOCKER_IMAGE = 1
-  MYSQL_DUMP = 2
-
-  def to_yaml(io)
-    to_s(io)
-  end
-end
-
-class LocalFileConfig
-  YAML.mapping(
-    name: String,
-    type: LocalType, # enum ?
-    path: String
-  )
-end
-
-class LocalDockerConfig
-  YAML.mapping(
-    name: String,
-    type: LocalType, # enum ?
-    docker_image: String
-  )
-end
-
-alias LocalConfig =
-  LocalFileConfig |
-  LocalDockerConfig
-

src/deployment.cr

@@ -1,4 +1,62 @@
 
-require "./deployment/docker_image_to_dokku_app"
+require "./deployment/*"
-require "./deployment/mysql_dump_to_dokku_mariadb"
+# run_script_to_dokku_app"
+# require "./deployment/transfer_docker_image_to_dokku_app"
+# require "./deployment/transfer_mysql_dump_to_dokku_mariadb"
+
+
+class Deployment
+  def self.apply_config!(config)
+    config.deployments.each do |deployment_config|
+      Deployment.apply_deployment!(config, deployment_config)
+    end
+  end
+
+  def self.apply_deployment!(config : Config, deployment_config : TransferDeploymentConfig) 
+    src = config.find_endpoint(deployment_config.transfer.src)
+    dest = config.find_endpoint(deployment_config.transfer.dest)
+
+    puts "Trying TransferDeploymentConfig: #{src.class} --> #{dest.class}..."
+    self.apply_transfer!(config, src, dest)
+  end
+
+  def self.apply_deployment!(config : Config, deployment_config : RunDeploymentConfig) 
+    src = config.find_endpoint(deployment_config.run.src)
+    dest = config.find_endpoint(deployment_config.run.dest)
+    puts "Trying RunDeploymentConfig: #{src.class} --> #{dest.class}..."
+
+    self.apply_run!(config, src, dest)
+  end
+
+  def self.apply_run!(config, src, dest)
+    puts "WARNING: run #{src.class} --> #{dest.class} missing!".colorize(:yellow)
+  end
+
+  def self.apply_transfer!(config, src, dest)
+    puts "WARNING: transfer #{src.class} --> #{dest.class} missing!".colorize(:yellow)
+  end
+
+  def something
+    deployment = 
+      case deployment_config
+      when DokkuAppDeploymentConfig then
+        DockerImageToDokkuAppDeployment.new(
+          local.as(DockerImageLocalConfig), 
+          remote, 
+          deployment_config.as(DokkuAppDeploymentConfig)
+        )
+      when MysqlDumpToDokkuMariadbDeployment then
+        DeploymentMariadb.new(
+          local.as(MysqlDumpLocalConfig), 
+          remote, 
+          deployment_config.as(DokkuMariadbDeploymentConfig)
+        )
+      when Nil
+        nil
+      end
+
+      next if deployment.nil?
+      deployment.run
+  end
+end
 

src/deployment/docker_image_to_dokku_app.cr

@@ -1,107 +0,0 @@
-
-require "colorize"
-
-class DeploymentApp
-  def initialize(@local : LocalDockerConfig, @remote : RemoteConfig, @deployment : DeploymentAppConfig)
-  end
-
-  def run
-    dokku_app = @deployment.as(DeploymentAppConfig).dokku_app
-    app = dokku_app.name
-    image_meta = image_tag(@local.docker_image, app)
-    image_push(@remote.host, image_meta["tag_name_version"])
-    image_deploy(@remote.host, image_meta["app"], image_meta["version"])
-  end
-
-  # private def image_tag(docker_compose_yml : String, service : String, app : String)
-  #   version = `date +"v%Y%m%d_%H%M"`.strip
-  #   tag_name = "dokku/#{app}"
-  #   tag_name_version = "#{tag_name}:#{version}"
-  #   image = `docker-compose -f #{docker_compose_yml} images -q #{service} `.strip
-  #   Process.run "docker", ["tag", image, tag_name_version]
-
-  #   res = {
-  #     app: app,
-  #     version: version,
-  #     tag_name_version: tag_name_version
-  #   }
-  #   puts YAML.dump({ image_tag: res })
-  #   puts "---"
-  #   return res
-  # end
-
-  private def image_tag(docker_image : String, app : String)
-    version = `date +"v%Y%m%d_%H%M"`.strip
-    tag_name = "dokku/#{app}"
-    tag_name_version = "#{tag_name}:#{version}"
-    Process.run "docker", ["tag", docker_image, tag_name_version]
-
-    res = {
-      app: app,
-      version: version,
-      tag_name_version: tag_name_version
-    }
-    puts YAML.dump({ image_tag: res })
-    puts "---"
-    return res
-  end
-
-  private def image_push(host, tag_name_version)
-    # docker save "$TAG_NAME_VERSION" \
-    #     | gzip \
-    #     | ssh "$HOST_REMOTE" "gunzip | docker load"
-
-    pipe1_reader, pipe1_writer = IO.pipe(true)
-    pipe2_reader, pipe2_writer = IO.pipe(true)
-
-    p3_out = IO::Memory.new
-    puts "Pushing image...".colorize(:yellow)
-    p3 = Process.new "ssh", [host, "gunzip | docker load"], 
-      input: pipe2_reader, output: p3_out, error: STDERR
-
-    p2 = Process.new "gzip",
-      input: pipe1_reader, 
-      output: pipe2_writer,
-      error: STDERR
-
-    p1 = Process.new "docker", ["save", tag_name_version], 
-      output: pipe1_writer, 
-      error: STDERR
-
-    status = p1.wait
-    pipe1_writer.close
-    if status.success? 
-      puts "-----> Docker image successfully exported"
-    else 
-      STDERR.puts "Error (code #{status.exit_status}) when exporting docker image!"
-      exit 1
-    end
-
-    status = p2.wait
-    pipe1_reader.close
-    pipe2_writer.close
-    if ! status.success?
-      STDERR.puts "Error (code #{status.exit_status}) when gzipping image!"
-    end
-
-    status = p3.wait
-    pipe2_reader.close
-    if status.success?
-      puts "-----> Docker image successfully imported on #{host}"
-    else
-      STDERR.puts "Error (code #{status.exit_status}) when importing docker image!"
-    end
-    puts "Image pushed successfully!".colorize(:green)
-  end
-
-  private def image_deploy(host, app, version)
-    puts "Deploying image #{app}:#{version}...".colorize(:yellow)
-    status = Process.run "ssh", [host, "dokku tags:deploy #{app} #{version}"],
-      output: STDOUT, error: STDOUT
-    if status.success?
-      puts "Image deployed successfully!".colorize(:green)
-    else
-      STDERR.puts "Error (code #{status.exit_status}) when deploying image!"
-    end
-  end
-end

src/deployment/run_script_to_dokku_app.cr

@@ -0,0 +1,15 @@
+
+require "colorize"
+
+class Deployment
+  def self.apply_run!(
+    config : Config, 
+    src : ScriptEndpointConfig, 
+    dest : DokkuAppEndpointConfig 
+  )
+    dokku_app = dest.name 
+  #  image_meta = image_tag(@local.docker_image, dokku_app)
+  #  image_push(@remote.host, image_meta["tag_name_version"])
+  #  image_deploy(@remote.host, image_meta["app"], image_meta["version"])
+  end
+end

src/deployment/transfer_docker_image_to_dokku_app.cr

@@ -0,0 +1,125 @@
+
+require "colorize"
+
+class Deployment
+  # puts YAML.dump({ image_tag: res })
+  # puts "---"
+
+  def self.apply_transfer!(
+    config : Config, 
+    src : DockerImageEndpointConfig, 
+    dest : DokkuAppEndpointConfig 
+  )
+    src_host = config.find_host(src.host)
+    dest_host = config.find_host(dest.host)
+
+    pp src 
+    dest_docker_image = build_docker_image_ref_timestamp(dest.name)
+    docker_image_tag!(src, dest_docker_image)
+    docker_image_push!(dest, src, dest_docker_image, dest_host)
+    # docker_image_deploy!(dest_host, dest_docker_image, dest_host)
+  end
+
+  private def self.build_docker_image_ref_timestamp(
+    dokku_app : String
+  )
+    dest_ref = DockerImageEndpointConfigSettings.new(
+      name: "dokku/#{dokku_app}",
+      tag: `TZ=UTC date +"v%Y%m%d_%H%M"`.strip
+    )
+    return dest_ref
+  end
+
+
+ #  private def self.docker_image_push!(
+ #  )
+ #    STDERR.puts "ERROR: pushing docker images to local host is not implemented"
+ #    exit 2
+ #  end
+
+
+  private def self.docker_image_tag!(
+    src : DockerImageEndpointConfig, 
+    dest_docker_image : DockerImageEndpointConfigSettings, 
+  )
+    puts "Tagging image...".colorize(:yellow)
+    puts YAML.dump({ 
+      src: src.docker_image.to_s, 
+      dest: dest_docker_image.to_s 
+    })
+  end
+
+  # docker tag ... dokku/...:tag
+  # docker save "$TAG_NAME_VERSION" \
+  #     | gzip \
+  #     | ssh "$HOST_REMOTE" "gunzip | docker load"
+  private def self.docker_image_push!(
+    dest : DokkuAppEndpointConfig,
+    src : DockerImageEndpointConfig, 
+    dest_docker_image : DockerImageEndpointConfigSettings, 
+    dest_host : HostConfig,
+  )
+    if dest_host.is_a? LocalHostConfig 
+      return
+    end
+
+    Process.run "docker", ["tag", src.docker_image.to_s, dest_docker_image.to_s]
+
+    pipe1_reader, pipe1_writer = IO.pipe(true)
+    pipe2_reader, pipe2_writer = IO.pipe(true)
+
+    p3_out = IO::Memory.new
+    puts "Pushing image...".colorize(:yellow)
+    p3 = Process.new "ssh", [dest_host.ssh.to_s, "gunzip | docker load"], 
+      input: pipe2_reader, output: p3_out, error: STDERR
+
+    p2 = Process.new "gzip",
+      input: pipe1_reader, 
+      output: pipe2_writer,
+      error: STDERR
+
+    p1 = Process.new "docker", ["save", src.docker_image.to_s], 
+      output: pipe1_writer, 
+      error: STDERR
+
+    status = p1.wait
+    pipe1_writer.close
+    if status.success? 
+      puts "-----> Docker image successfully exported"
+    else 
+      STDERR.puts "Error (code #{status.exit_status}) when exporting docker image!"
+      exit 1
+    end
+
+    status = p2.wait
+    pipe1_reader.close
+    pipe2_writer.close
+    if ! status.success?
+      STDERR.puts "Error (code #{status.exit_status}) when gzipping image!"
+    end
+
+    status = p3.wait
+    pipe2_reader.close
+    if status.success?
+      puts "-----> Docker image successfully imported on #{dest_host.ssh.to_s}"
+    else
+      STDERR.puts "Error (code #{status.exit_status}) when importing docker image!"
+    end
+    puts "Image pushed successfully!".colorize(:green)
+  end
+
+  private def self.docker_image_deploy!(
+    src : DockerImageEndpointConfig, 
+    dest : DockerImageEndpointConfig,
+    dest_docker_image : DockerImageEndpointConfigSettings, 
+  )
+    puts "Deploying image #{app}:#{version}...".colorize(:yellow)
+    status = Process.run "ssh", [dest.host.to_s, "dokku tags:deploy #{app} #{version}"],
+      output: STDOUT, error: STDOUT
+    if status.success?
+      puts "Image deployed successfully!".colorize(:green)
+    else
+      STDERR.puts "Error (code #{status.exit_status}) when deploying image!"
+    end
+  end
+end

src/deployment/transfer_mysql_dump_to_dokku_mariadb.cr

@@ -1,13 +1,15 @@
 
-class DeploymentMariadb
+class Deployment
-  def initialize(@local : LocalFileConfig, @remote : RemoteConfig, @deployment : DeploymentMariadbConfig)
-  end
 
-  def run
+  def self.apply_transfer!(
-    dokku_mariadb = @deployment.dokku_mariadb.as(DokkuMariadbConfig)
+    config : Config, 
-    local_path = @local.path
+    src : MysqlDumpEndpointConfig, 
+    dest : DokkuMariadbEndpointConfig 
+  )
+    dokku_mariadb = dest.name
+    # local_path = @local.path
     # puts @local.inspect
-    file_push(@remote.host, local_path, dokku_mariadb.name)
+    # file_push(@remote.host, local_path, dokku_mariadb.name)
   end
 
   private def file_push(host, local_path, dokku_mariadb_name)

src/pushokku.cr

@@ -4,6 +4,7 @@ require "yaml"
 require "colorize"
 
 require "./config"
+require "./validation"
 require "./deployment"
 
 module Pushokku
@@ -81,35 +82,9 @@ module Pushokku
       config = app.load_config(opts["config_file"])
       # env_config = App.get_config(config, opts["environment"])
 
-      config.deployments.each do |deployment_config|
+      Validation.validate_config!(config)
-        local = config.locals.select { |l| l.name == deployment_config.local }.first
+      Deployment.apply_config!(config)
-        remote = config.remotes.select { |r| r.name == deployment_config.remote }.first
+      exit 0
-        if local.nil?
-          puts "Unknown local #{deployment_config.local}. Exiting."
-          exit 2
-        end
-        if remote.nil?
-          puts "Unknown remote #{deployment_config.remote}. Exiting."
-          exit 2
-        end
-
-
-        deployment = 
-          case deployment_config
-          when DeploymentAppConfig then
-
-            DeploymentApp.new(local.as(LocalDockerConfig), remote, deployment_config.as(DeploymentAppConfig))
-          when DeploymentMariadbConfig then
-            DeploymentMariadb.new(local.as(LocalFileConfig), remote, deployment_config.as(DeploymentMariadbConfig))
-          when Nil
-            nil
-          end
-
-        next if deployment.nil?
-        deployment.run
-      end
-
-      exit 2
     end
   end
 end

src/validation.cr

@@ -0,0 +1,14 @@
+
+class Validation
+  def self.validate_config!(config)
+    pp config
+    config.deployments.each do |deployment_config|
+      Validation.validate_deployment!(config, deployment_config)
+    end
+  end
+
+  def self.validate_deployment!(config : Config, deployment : DeploymentConfig)
+    STDERR.puts "WARNING: validation for #{deployment.class} not yet implemented".colorize(:yellow)
+  end
+end
+