ci: Rename service docker

.drone.yml

@@ -0,0 +1,95 @@
+---
+kind: pipeline
+type: docker
+name: "stage:build"
+
+trigger:
+  event:
+    - push
+    - pull_request
+
+services:
+  - name: service:docker
+    image: docker:dind
+    privileged: true
+    volumes:
+      - name: dockersock
+        path: /var/run
+
+volumes:
+  - name: crystal_cache
+    temp: {}
+  - name: docker_cache
+    temp: {}
+  - name: dockersock
+    temp: {}
+
+steps:
+  - name: "build:prepare"
+    image: busybox
+    commands:
+      # Debug
+      - pwd
+      - mkdir -p _artifacts _cache
+
+  - name: "build:executable"
+    image: crystallang/crystal:1.5.0
+    volumes:
+      - name: crystal_cache
+        path: /drone/src/_cache
+    commands:
+      # Debug
+      - pwd
+      - crystal version
+      - shards version
+      # Build
+      - make build
+
+  - name: "build:docker-image"
+    image: docker:20.10.16-dind
+    volumes:
+      - name: dockersock
+        path: /var/run
+      - name: docker_cache
+        path: /drone/src/_cache
+    commands:
+      # Prepare
+      - apk add make
+      - test -f _cache/docker-image.tar && docker load < _cache/docker-image.tar
+      # Build
+      - make docker-build
+      - make docker-save > _artifacts/docker-image.tar
+      - make docker-save > _cache/docker-image.tar
+
+  - name: "publish:docker-image"
+    image: docker:20.10.16-dind
+    volumes:
+      - name: dockersock
+        path: /var/run
+    commands:
+      # Prepare
+      - test -f _artifacts/docker-image.tar
+      - apk add make
+      - make docker-load < _artifacts/docker-image.tar
+      # Build
+      - make docker-push
+
+---
+kind: pipeline
+type: docker
+name: "stage:deploy"
+
+trigger:
+  event:
+    - promote
+  target:
+    - production
+
+steps:
+  - name: "deploy:prepare"
+    image: busybox
+    commands:
+      # Debug
+      - echo "No op"
+
+#

.gitignore

@@ -1,2 +1,2 @@
-lib
+/lib
-bin
+/bin

.gitlab-ci.yml

@@ -7,11 +7,59 @@
 # Note that environment variables can be set in several places
 # See https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
 stages:
+  - build
   - test
+  - publish
+
+build:executable:
+  stage: build
+  image: crystallang/crystal:1.5.0
+  script:
+    # Debug
+    - crystal version
+    - shards version
+    # Build
+    - make build
+  artifacts:
+    paths:
+      - bin
+  cache:
+    paths:
+      - lib
+
+build:docker-image:
+  stage: build
+  image: docker:20.10.16
+  services:
+    - docker:20.10.16-dind
+  needs:
+    - build:executable
+  script:
+    - apk install make
+    - make docker-build
+    - make docker-save > _artifacts/docker-image.tar
+  artifacts:
+    paths:
+      - _artifacts
 
 sast:
   stage: test
 
+publish:docker-image:
+  stage: publish
+  needs:
+    - build:docker-image
+  image: docker:20.10.16
+  services:
+    - docker:20.10.16-dind
+  script:
+    - apk install make
+    - make docker-load < _artifacts/docker-image.tar
+    - make docker-push
+  artifacts:
+    paths:
+      - bin
+
 include:
   - template: Security/SAST.gitlab-ci.yml
 

Makefile

@@ -1,15 +1,25 @@
 
+DOCKER_IMAGE=glenux/openstack
+
+all: build
 
 build:
-	shards build
+	shards install
+	shards build --error-trace
 
 docker: docker-build docker-test docker-push
 
 docker-build:
-	docker build -t glenux/openstack-arkisto .
+	docker build --file docker/Dockerfile -t $(DOCKER_IMAGE) .
 
 docker-push:
-	docker push glenux/openstack-arkisto
+	docker push $(DOCKER_IMAGE)
 
 docker-test:
-	docker run glenux/openstack-arkisto arkisto --version
+	docker run $(DOCKER_IMAGE) arkisto --version
+
+docker-save:
+	docker save $(DOCKER_IMAGE)
+
+docker-load:
+	docker load

README.md

@@ -1,4 +1,38 @@
+
+
 # Arkisto
 
-A simple tool to automate openstack snapshots
+[![Build Status](https://cicd.apps.glenux.net/api/badges/glenux/arkisto/status.svg?ref=refs/heads/develop)](https://cicd.apps.glenux.net/glenux/arkisto)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Donate on patreon](https://img.shields.io/badge/patreon-donate-orange.svg)](https://patreon.com/glenux)
+
+A simple tool to automate and manage openstack snapshots 
+
+## Installation
+
+### With docker
+
+Make sure you have Docker and Make installed. Then type
+
+    make docker-build
+
+### Manual
+
+Make sure you have installed the following software on your system:
+
+* Crystal
+* Make
+* libevent-core-2.1-7
+* python-openstackclient
+
+Then type
+
+    make build
+
+## Usage
+
+## Licence
+
+
+
 

docker/Dockerfile

@@ -11,27 +11,30 @@ FROM python:3.7-bullseye AS runner
 
 RUN apt-get update \
  && apt-get -yq install \
- 	libevent-core-2.1-7 \
+      gettext \
-    gettext \
+      jq \
-    jq \
+      libevent-core-2.1-7 \
-    python3-aodhclient \
+      zip \
-    python3-barbicanclient \
-    python3-ceilometerclient \
-    python3-cinderclient \
-    python3-cloudkittyclient \
-    python3-designateclient \
-    python3-gnocchiclient \
-    python3-octaviaclient \
-    python3-osc-placement \
-    python3-openstackclient \
-    python3-pankoclient \
-    zip \
  && rm -rf /var/lib/apt/lists/*
 
+RUN pip install \
+      aodhclient \
+      cinder \
+      gnocchiclient \
+      osc-placement \
+      panko \
+      python-barbicanclient \
+      python-ceilometerclient \
+      python-cloudkittyclient \
+      python-designateclient \
+      python-octaviaclient \
+      python-openstackclient
+
+
 # RUN wget https://github.com/sapcc/cyclone/releases/download/v0.1.28/cyclone \
 #      -O /usr/bin/cyclone \
 #  && chmod +x /usr/bin/cyclone
 
 COPY --from=builder /app/bin/arkisto /usr/bin/arkisto
 
-CMD /bin/bash
+CMD ["/bin/bash"]

src/lib/actions.cr

@@ -0,0 +1,26 @@
+
+require "tablo"
+
+module Arkisto
+  class Action
+    def initialize(_config : ConfigModel, _options : ActionOptions)
+      # Do nothing by default
+      return
+    end
+
+    def perform
+      # Do nothing by default
+      return
+    end
+  end
+
+  class ActionFactory
+    def self.build(action_type : Action.class, config, options)
+      action = action_type.new(config, options)
+    end
+  end
+end
+
+require "./actions/*"
+
+

src/lib/actions/apply.cr

@@ -0,0 +1,91 @@
+
+require "../openstack"
+require "../actions"
+
+module Arkisto
+  class ApplyAction < Action
+    def initialize(config : ConfigModel, options : ActionOptions)
+      puts "action = Apply"
+      @config = config
+    end
+
+    # First create today's backup
+    # Then remove old backups
+    def perform
+      _create_backup_all
+      _delete_backup_all
+      return
+    end
+
+    def _create_backup_all
+      puts "_create_backup_all".colorize(:yellow)
+      os_volumes = OpenStack.volume_list
+
+      @config.targets.each do |config_volume_item|
+        os_volume_item = 
+          os_volumes.select {|vol| vol.id == config_volume_item.volume_id }.first?
+
+        if os_volume_item.nil?
+          puts "  ERROR: no volume found with this id".colorize(:red)
+          exit(1)
+        end
+
+        _create_backup(config_volume_item, os_volume_item)
+      end
+    end
+
+    def _create_backup(config_item : TargetItemModel, volume_item : OSVolume::ListItemModel)
+      puts "_create_backup".colorize(:yellow)
+      puts "+ openstack volume snapshot #{config_item.name.colorize(:yellow)} {"
+      puts "+   volume_id #{config_item.volume_id}"
+      puts "+   name #{volume_item.name}"
+      puts "+   size #{volume_item.size}"
+      puts "+ }"
+
+      created_volume = OpenStack.volume_snapshot_create(volume_item, config_item.name)
+    end
+
+    def _delete_backup_all
+      puts "_delete_backup_all".colorize(:yellow)
+      os_volumes = OpenStack.volume_list
+      os_all_snapshots = OpenStack.volume_snapshot_list
+
+      @config.targets.each do |config_volume_item|
+        os_volume_item = os_volumes.select {|vol| 
+          vol.id == config_volume_item.volume_id 
+        }.try &.first
+
+        os_volume_snapshots = os_all_snapshots.select {|snap| 
+          snap.volume == config_volume_item.volume_id
+        }
+
+        os_volume_snapshots.each do |snap_item|
+          _delete_backup(config_volume_item, snap_item)
+          pp snap_item
+        end
+
+        # _delete_backup(config_volume_item, os_volume_snapshot_item)
+        # volume_snapshots.each do |target_snapshot|
+        # os_volume_item = 
+        # os_volumes.select {|vol| vol.id == config_volume_item.volume_id }.first
+      end
+    end
+
+    def _delete_backup(config_item : TargetItemModel, snap_item : OSVolume::SnapshotListItemModel)
+      puts "_delete_backup".colorize(:yellow)
+
+      puts "- openstack volume snapshot #{config_item.name.colorize(:yellow)} {"
+      puts "-   volume_id #{config_item.volume_id}"
+      puts "-   created_at #{snap_item.created_at}"
+      puts "-   snap_id #{snap_item.id}"
+      puts "-   name #{snap_item.name}"
+      puts "-   size #{snap_item.size}"
+      puts "- }"
+
+      pp config_item
+      pp snap_item
+
+      # FIXME: not implemented
+    end
+  end
+end

src/lib/actions/none.cr

@@ -0,0 +1,17 @@
+
+require "../actions"
+
+module Arkisto
+  class NoneAction < Action
+    def initialize(config : ConfigModel, options : ActionOptions)
+      puts "action = None"
+      # @global_options = global_options
+      # @config = config
+    end
+
+    def perform
+      # Do nothing by default
+      return
+    end
+  end
+end

src/lib/actions/plan.cr

@@ -0,0 +1,21 @@
+
+require "../actions"
+
+module Arkisto
+  class PlanAction < Action
+    def initialize(config : ConfigModel, options : ActionOptions)
+      @config = config
+      puts "action = Plan"
+    end
+
+    def perform
+      @config.targets.each do |target|
+        puts "#{target.name}"
+        puts "- volume_id #{target.volume_id}"
+        # puts "- snapshot_prefix #{target.snapshot_prefix}"
+        puts ""
+      end
+      return
+    end
+  end
+end

src/lib/models.cr

@@ -0,0 +1,11 @@
+
+require "yaml"
+
+module Arkisto
+  class Model
+    include YAML::Serializable
+  end
+end
+
+require "./types"
+require "./models/*"

src/lib/models/config.cr

@@ -0,0 +1,11 @@
+
+require "../models"
+
+module Arkisto
+    class ConfigModel < Model
+      property version  : String
+      property retention : RetentionModel
+      property targets : Array(TargetItemModel)
+  end
+end
+

src/lib/models/os_vol_list_item.cr

@@ -0,0 +1,74 @@
+
+
+# - Attached to: []
+#   ID: 8c87c196-1dd6-4190-bff5-00b543965093
+#   Name: ovh-managed-kubernetes-e6j07m-pvc-9fad19b4-b32c-452a-9893-524fcdc3a4b1
+#   Size: 50
+#   Status: available
+# - Attached to:
+#   - attached_at: '2022-06-27T17:38:53.000000'
+#     attachment_id: 88a7dfc4-5bdf-4864-b38d-c82169c37b9d
+#     device: /dev/sdc
+#     host_name: null
+#     id: c9b11fcb-a6a4-4387-b55b-34121f97dac4
+#     server_id: 37220c33-19e7-45cc-9c6a-ddf957be3c76
+#     volume_id: c9b11fcb-a6a4-4387-b55b-34121f97dac4
+#   ID: c9b11fcb-a6a4-4387-b55b-34121f97dac4
+#   Name: ovh-managed-kubernetes-e6j07m-pvc-13dcac68-71c9-4794-b0ab-d48e76d843ef
+#   Size: 5800
+#   Status: in-use
+
+## LONG
+
+# - Attached to:
+#   - attached_at: '2022-06-27T17:31:53.000000'
+#     attachment_id: 845a7ff2-ad54-4040-b69b-fce7fdfdece2
+#     device: /dev/sdb
+#     host_name: null
+#     id: 6d4492e5-6b38-4ad3-b9bf-27ba86e2506b
+#     server_id: 37220c33-19e7-45cc-9c6a-ddf957be3c76
+#     volume_id: 6d4492e5-6b38-4ad3-b9bf-27ba86e2506b
+#   Bootable: 'false'
+#   ID: 6d4492e5-6b38-4ad3-b9bf-27ba86e2506b
+#   Name: ovh-managed-kubernetes-e6j07m-pvc-7bf995bf-ac82-4f42-94ca-78cd9c78959d
+#   Properties:
+#     attached_mode: rw
+#     cinder.csi.openstack.org/cluster: kubernetes
+#     readonly: 'False'
+#   Size: 1
+#   Status: in-use
+#   Type: high-speed
+
+
+require "../models"
+
+module Arkisto
+  module OSVolume
+    class ListItemModel < Model
+      @[YAML::Field(key: "Attached to")]
+      property attached_to : Array(OSVolume::ListItemAttachment::ListItemModel)
+
+      @[YAML::Field(key: "Bootable")]
+      property bootable : String
+
+      @[YAML::Field(key: "ID")]
+      property id : String
+
+      @[YAML::Field(key: "Name")]
+      property name : String
+
+      @[YAML::Field(key: "Properties")]
+      property properties : YAML::Any
+
+      @[YAML::Field(key: "Size")]
+      property size : UInt16
+
+      @[YAML::Field(key: "Status")]
+      property status : String
+
+      @[YAML::Field(key: "Type")]
+      property type : String
+    end
+  end
+end
+

src/lib/models/os_vol_list_item_attach_item.cr

@@ -0,0 +1,20 @@
+
+require "../models"
+
+module Arkisto
+  module OSVolume
+    class ListItemAttachment
+      class ListItemModel < Model
+        property attached_at : String
+        property attachment_id : String
+        property device : String
+        property host_name : String
+        property id : String
+        property server_id : String
+        property volume_id : String
+      end
+    end
+  end
+end
+
+

src/lib/models/os_vol_snap_create_item.cr

@@ -0,0 +1,42 @@
+
+# created_at: '2022-06-28T11:26:06.266927'
+# description: null
+# id: 52fa0022-3ed8-43bc-90d9-08d22597fb35
+# name: pouet
+# properties: {}
+# size: 50
+# status: creating
+# updated_at: null
+# volume_id: de7b118f-413b-4962-a028-a742df4bc4fc
+
+require "../models"
+
+module Arkisto
+  module OSVolume
+    class SnapshotCreateItemModel < Model
+      @[YAML::Field(key: "created_at")]
+      property created_at : String
+
+      @[YAML::Field(key: "description")]
+      property description : String
+
+      @[YAML::Field(key: "id")]
+      property id : String  ## UUID
+
+      @[YAML::Field(key: "name")]
+      property name : String
+
+      @[YAML::Field(key: "properties")]
+      property properties : YAML::Any
+
+      @[YAML::Field(key: "size")]
+      property size : UInt16
+
+      @[YAML::Field(key: "status")]
+      property status : String
+
+      @[YAML::Field(key: "volume_id")]
+      property volume_id : String
+    end
+  end
+end

src/lib/models/os_vol_snap_list_item.cr

@@ -0,0 +1,51 @@
+
+## volume snapshot list -f yaml
+
+# - Description: null
+#   ID: 86367f4d-fbc2-4e09-b0e7-1b4b73f72baa
+#   Name: db-snapshot01
+#   Size: 50
+#   Status: available
+
+## volume snapshot list --long -f yaml
+
+# - Created At: '2022-06-27T14:09:47.000000'
+#   Description: null
+#   ID: 86367f4d-fbc2-4e09-b0e7-1b4b73f72baa
+#   Name: db-snapshot01
+#   Properties: {}
+#   Size: 50
+#   Status: available
+#   Volume: 8c87c196-1dd6-4190-bff5-00b543965093
+
+require "../models"
+
+module Arkisto
+  module OSVolume
+    class SnapshotListItemModel < Model
+      @[YAML::Field(key: "Created At")]
+      property created_at : String
+
+      @[YAML::Field(key: "Description")]
+      property description : String
+
+      @[YAML::Field(key: "ID")]
+      property id : String  ## UUID
+
+      @[YAML::Field(key: "Name")]
+      property name : String
+
+      @[YAML::Field(key: "Properties")]
+      property properties : YAML::Any
+
+      @[YAML::Field(key: "Size")]
+      property size : UInt16
+
+      @[YAML::Field(key: "Status")]
+      property status : String
+
+      @[YAML::Field(key: "Volume")]
+      property volume : String
+    end
+  end
+end

src/lib/models/retention.cr

@@ -0,0 +1,10 @@
+
+require "../models"
+
+module Arkisto
+  class RetentionModel < Model
+    property days : UInt8
+    property cycle : Bool
+    property max_items : UInt8
+  end
+end

src/lib/models/target.cr

@@ -0,0 +1,11 @@
+
+require "../models"
+
+module Arkisto
+  class TargetItemModel < Model
+    property name : String
+    property volume_id : String
+    # property snapshot_prefix : String
+  end
+end
+

src/lib/openstack.cr

@@ -0,0 +1,77 @@
+
+module Arkisto
+  class OpenStack
+    #
+    # openstack volume list
+    #
+    def self.volume_list
+      cmd = "openstack"
+      args = [
+        "volume", "list",
+        "--long",
+        "--format", "yaml"
+      ]
+      stdout = IO::Memory.new
+      stderr = IO::Memory.new
+
+      status = Process.run(cmd, args: args, output: stdout, error: stderr)
+      if ! status.success?
+        raise RuntimeError.new("command failed for: #{cmd} #{args.join(" ")}")  # FIXME: improve message
+      end
+
+      return Array(OSVolume::ListItemModel).from_yaml(stdout.to_s)
+    rescue ex : RuntimeError
+      puts "ERROR: #{ex.message}".colorize(:red)
+      exit(1)
+    end
+
+    #
+    # openstack volume snapshot list
+    #
+    def self.volume_snapshot_list
+      cmd = "openstack"
+      args =[
+        "volume", "snapshot", "list",
+        "--long",
+        "--format", "yaml"
+      ]
+      stdout = IO::Memory.new
+      stderr = IO::Memory.new
+
+      status = Process.run(cmd, args: args, output: stdout, error: stderr)
+      if ! status.success?
+        raise RuntimeError.new("command failed for: #{cmd} #{args.join(" ")}")  # FIXME: improve message
+      end
+
+      return Array(OSVolume::SnapshotListItemModel).from_yaml(stdout.to_s)
+    rescue ex : RuntimeError
+      puts "ERROR: #{ex.message}".colorize(:red)
+      exit(1)
+    end
+
+    #
+    # openstack volume snapshot create --volume VOLUME_ID  --force SNAPSHOT_NAME -f yaml 
+    #
+    def self.volume_snapshot_create(target_volume, snap_name)
+      cmd = "openstack"
+      args = ["volume", "snapshot", "create",
+              "--volume", target_volume.id,
+              "--force",  
+              snap_name,
+              "--format", "yaml"]
+      stdout = IO::Memory.new
+      stderr = IO::Memory.new
+
+      status = Process.run(cmd, args: args, output: stdout, error: stderr)
+      if ! status.success?
+        raise RuntimeError.new("#{cmd} #{args.join(" ")}\n#{stderr.to_s}")  # FIXME: improve message
+      end
+
+      return OSVolume::SnapshotCreateItemModel.from_yaml(stdout.to_s)
+
+    rescue ex : RuntimeError
+      puts "ERROR: #{ex.message}".colorize(:red)
+      exit(1)
+    end
+  end
+end

src/lib/types.cr

@@ -0,0 +1,16 @@
+
+require "./actions"
+
+module Arkisto
+  alias GlobalOptions = {
+    action: Action.class,
+    verbose: Bool,
+    dry_run: Bool,
+    config_file: String
+  }
+
+  alias ActionOptions = {
+    verbose: Bool,
+    dry_run: Bool
+  }
+end

src/lib/version.cr

@@ -0,0 +1,7 @@
+module Arkisto
+  class Version
+    PROGRAM_ARKISTOCTL = "arkistoctl"
+    VERSION = "0.1.0"
+  end
+end
+