From d9a2a80b5f9f960f4a5586d16ad249e14efa5a27 Mon Sep 17 00:00:00 2001 From: Glenn Date: Fri, 12 Jul 2024 14:36:06 +0200 Subject: [PATCH] Initial import --- .travis.yml | 29 +++++ README.md | 38 ++++++ defaults/main.yml | 12 ++ files/duplicity_helper_for_dokku | 110 ++++++++++++++++++ handlers/main.yml | 2 + meta/main.yml | 53 +++++++++ tasks/config_duplicity.yml | 45 +++++++ tasks/config_gnupg.yml | 49 ++++++++ tasks/install_duplicity.yml | 45 +++++++ tasks/install_helpers.yml | 12 ++ tasks/main.yml | 22 ++++ tasks/prerequisites.yml | 13 +++ .../etc.glenux.duplicity.backends.json.j2 | 74 ++++++++++++ .../etc.glenux.duplicity.duplicity.conf.j2 | 3 + tests/inventory | 2 + tests/test.yml | 5 + vars/main.yml | 2 + 17 files changed, 516 insertions(+) create mode 100644 .travis.yml create mode 100644 README.md create mode 100644 defaults/main.yml create mode 100644 files/duplicity_helper_for_dokku create mode 100644 handlers/main.yml create mode 100644 meta/main.yml create mode 100644 tasks/config_duplicity.yml create mode 100644 tasks/config_gnupg.yml create mode 100644 tasks/install_duplicity.yml create mode 100644 tasks/install_helpers.yml create mode 100644 tasks/main.yml create mode 100644 tasks/prerequisites.yml create mode 100644 templates/etc.glenux.duplicity.backends.json.j2 create mode 100644 templates/etc.glenux.duplicity.duplicity.conf.j2 create mode 100644 tests/inventory create mode 100644 tests/test.yml create mode 100644 vars/main.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..36bbf62 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,29 @@ +--- +language: python +python: "2.7" + +# Use the new container infrastructure +sudo: false + +# Install ansible +addons: + apt: + packages: + - python-pip + +install: + # Install ansible + - pip install ansible + + # Check ansible version + - ansible --version + + # Create ansible.cfg with correct roles_path + - printf '[defaults]\nroles_path=../' >ansible.cfg + +script: + # Basic role syntax check + - ansible-playbook tests/test.yml -i tests/inventory --syntax-check + +notifications: + webhooks: https://galaxy.ansible.com/api/v1/notifications/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..225dd44 --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +Role Name +========= + +A brief description of the role goes here. + +Requirements +------------ + +Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. + +Role Variables +-------------- + +A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. + +Dependencies +------------ + +A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + - hosts: servers + roles: + - { role: username.rolename, x: 42 } + +License +------- + +BSD + +Author Information +------------------ + +An optional section for the role authors to include contact information, or a website (HTML is not allowed). diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100644 index 0000000..9ff3378 --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,12 @@ +--- +# defaults file for glenux.duplicity +duplicity_enable_dokku: true +duplicity_enable_folders: false + +duplicity_gpg_secret_key: null +duplicity_gpg_passphrase: null + +# duplicity_hostname: null +# duplicity_work_directory: null + +# diff --git a/files/duplicity_helper_for_dokku b/files/duplicity_helper_for_dokku new file mode 100644 index 0000000..df2b149 --- /dev/null +++ b/files/duplicity_helper_for_dokku @@ -0,0 +1,110 @@ +#!/bin/sh +# vim: set ts=2 sw=2 et: + +set -u +set -e + +APP_DOKKU_ROOT=/home/dokku +APP_VOLUMES_ROOT=/var/lib/dokku/data/storage +APP_SERVICE_ROOT=/var/lib/dokku/services + +if [ -r /etc/duplicity/config ]; then + . /etc/duplicity/config +else + echo "Unable to read config file" +fi +DUPLICITY_HOSTNAME="${DUPLICITY_HOSTNAME:-}" +DUPLICITY_WORK_DIR="${DUPLICITY_WORK_DIR:-}" +DUPLICITY_ENABLE="${DUPLICITY_ENABLE:-}" + +make_archive() { + app="$1" + dir="$2" + age="7D" + + echo "Creating backup for $app..." + echo "- data dir $dir" + + if [ ! -d "$dir" ]; then + >&2 echo "ERROR: missing data dir $dir" + exit 1 + fi + if [ "$DUPLICITY_ENABLE" != "true" ]; then + echo "- dryrun mode (skipping the real work)" + return + fi + duplicity \ + --file-prefix-manifest "hot_${app}_" \ + --file-prefix-signature "hot_${app}_" \ + --file-prefix-archive "cold_${app}_" \ + --full-if-older-than "$age" \ + "$dir" \ + 'multi:///root/.duplicity/config.json?mode=mirror&onfail=abort' +} + +clean_archive() { + age="1D" + echo "Removing old backups (> $age)" + if [ "$DUPLICITY_ENABLE" != "true" ]; then + echo "- dryrun mode (skipping the real work)" + return + fi + duplicity \ + remove-older-than "$age" \ + --force 'multi:///root/.duplicity/config.json?mode=mirror&onfail=abort' +} + +## VALIDATE INPUT + +if [ -z "$DUPLICITY_HOSTNAME" ]; then + >&2 echo "ERROR: Variable DUPLICITY_HOSTNAME must be declared in /etc/duplicity/config" + exit 1 +fi + +if [ -z "$DUPLICITY_WORK_DIR" ]; then + >&2 echo "ERROR: Variable DUPLICITY_WORK_DIR must be declared in /etc/duplicity/config" + exit 1 +fi + +PASSPHRASE="$(cat /etc/duplicity/gpg_backup_passphrase)" +TMPDIR="$DUPLICITY_WORK_DIR" +# /mnt/one.data.infra.glenux.net/duplicity + +export PASSPHRASE +export TMPDIR + + +## DOKKU DATA +for app in $( dokku apps:list |tail -n+2 |xargs echo ) ; do + NAME="DOKKU_${app}@${DUPLICITY_HOSTNAME}" + DATA_DIR="$APP_DOKKU_ROOT/$app" + + make_archive "$NAME" "$DATA_DIR" +done + +## VOLUMES DATA +for app in $( dokku apps:list |tail -n+2 |xargs echo ) ; do + NAME="VOLUMES_${app}@${DUPLICITY_HOSTNAME}" + DATA_DIR="$APP_VOLUMES_ROOT/$app" + + make_archive "$NAME" "$DATA_DIR" +done + +## MARIADB DATA +for app in $( dokku mariadb:list |tail -n+2 |awk '{ print $1; }' |xargs echo ) ; do + NAME="MARIADB_$(dokku mariadb:list |grep "^$app " |awk '{ print $1 "@" $5; }')" + DATA_DIR="$APP_SERVICE_ROOT/mariadb/$app" + + make_archive "$NAME" "$DATA_DIR" +done + +## POSTGRES DATA +for app in $( dokku postgres:list |tail -n+2 |awk '{ print $1; }' |xargs echo ) ; do + NAME="POSTGRES_$(dokku postgres:list |grep "^$app " |awk '{ print $1 "@" $5; }')" + DATA_DIR="$APP_SERVICE_ROOT/postgres/$app" + + make_archive "$NAME" "$DATA_DIR" +done + +clean_archive + diff --git a/handlers/main.yml b/handlers/main.yml new file mode 100644 index 0000000..ec9102d --- /dev/null +++ b/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for glenux.duplicity \ No newline at end of file diff --git a/meta/main.yml b/meta/main.yml new file mode 100644 index 0000000..227ad9c --- /dev/null +++ b/meta/main.yml @@ -0,0 +1,53 @@ +galaxy_info: + author: your name + description: your role description + company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: license (GPL-2.0-or-later, MIT, etc) + + min_ansible_version: 2.9 + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + # + # Provide a list of supported platforms, and for each platform a list of versions. + # If you don't wish to enumerate all versions for a particular platform, use 'all'. + # To view available platforms and versions (or releases), visit: + # https://galaxy.ansible.com/api/v1/platforms/ + # + # platforms: + # - name: Fedora + # versions: + # - all + # - 25 + # - name: SomePlatform + # versions: + # - all + # - 1.0 + # - 7 + # - 99.99 + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. + \ No newline at end of file diff --git a/tasks/config_duplicity.yml b/tasks/config_duplicity.yml new file mode 100644 index 0000000..dc8743b --- /dev/null +++ b/tasks/config_duplicity.yml @@ -0,0 +1,45 @@ +--- +## +## Config: Duplicity Part +## + +- name: Create duplicity config directory + file: + path: /etc/glenux.duplicity + state: directory + mode: '0700' + owner: root + group: root + +- name: Install duplicity config file with variables + template: + src: templates/etc.glenux.duplicity.duplicity.conf.j2 + dest: /etc/glenux.duplicity/duplicity.conf + mode: "0600" + owner: root + group: root + +- name: Install duplicity config file with backends + template: + src: templates/etc.glenux.duplicity.backends.json.j2 + dest: /etc/glenux.duplicity/backends.json + mode: "0600" + owner: root + group: root + +- name: Install duplicity helper script for dokku + copy: + src: files/duplicity_helper_for_dokku + dest: /usr/sbin/duplicity_helper_for_dokku + mode: "0755" + owner: root + group: root + +- name: Create duplicity work directory + file: + path: "{{duplicity_work_directory}}" + state: directory + mode: '0700' + owner: root + group: root +# diff --git a/tasks/config_gnupg.yml b/tasks/config_gnupg.yml new file mode 100644 index 0000000..52df62a --- /dev/null +++ b/tasks/config_gnupg.yml @@ -0,0 +1,49 @@ +--- +## +## Config: GPG Part +## + +- name: Create GPG directory + file: + path: /root/.gnupg + state: directory + mode: '0700' + +- name: Send GPG passphrase + copy: + src: "{{duplicity_gpg_passphrase}}" + dest: /etc/glenux.duplicity/gnupg_backup.passphrase + mode: "0600" + +- name: Send GPG secret key + copy: + src: "{{duplicity_gpg_key}}" + dest: /etc/glenux.duplicity/gnupg_backup.key + mode: "0600" + +- name: Send GPG public key + copy: + src: "{{duplicity_gpg_pubkey}}" + dest: /etc/glenux.duplicity/gnupg_backup.pubkey + mode: "0600" + +- name: Import GPG key + command: "gpg --batch + --passphrase-file /etc/glenux.duplicity/gnupg_backup.passphrase + --pinentry-mode loopback + --import /etc/glenux.duplicity/gnupg_backup.key" + +- name: De-armor GPG key + command: "gpg -no-tty + --dearmor + /etc/glenux.duplicity/gnupg_backup.key" + +- name: Set GPG key trustlevel + shell: "gpg --with-fingerprint + --no-default-keyring + --secret-keyring /etc/glenux.duplicity/gnupg_backup.key.gpg + --list-secret-keys + --with-colons + | sed -E -n -e 's/^fpr:::::::::([0-9A-F]+):$/\\1:6:/p' + | gpg --import-ownertrust" +# diff --git a/tasks/install_duplicity.yml b/tasks/install_duplicity.yml new file mode 100644 index 0000000..10cb669 --- /dev/null +++ b/tasks/install_duplicity.yml @@ -0,0 +1,45 @@ +--- +## +## Duplicity +## + +# FIXME: test if duplicity is installed +# FIXME: test if duplicity is the right version + +- name: Download duplicity 0.8.18 + get_url: + url: https://launchpadlibrarian.net/516340710/duplicity-0.8.18.tar.gz + dest: /usr/src/duplicity-0.8.18.tar.gz + checksum: sha256:2643fea0f52920a0fb114069c78389f9621f1c24db7f26bda77bbc239b01ae53 + +- name: Extract duplicity 0.8.18 + unarchive: + remote_src: "yes" + src: /usr/src/duplicity-0.8.18.tar.gz + dest: /usr/src + creates: /usr/src/duplicity-0.8.18 + +- name: Install duplicity setuptools + shell: + cmd: pip3 install setuptools wheel + chdir: /usr/src/duplicity-0.8.18 + +- name: Install duplicity requirements + shell: + cmd: pip3 install -r requirements.txt + chdir: /usr/src/duplicity-0.8.18 + +- name: Install duplicity + shell: + cmd: python3 setup.py install + chdir: /usr/src/duplicity-0.8.18 + + +- name: Create duplicity work directory + file: + path: "{{duplicity_work_directory}}" + state: directory + mode: '0700' + owner: root + group: root +# diff --git a/tasks/install_helpers.yml b/tasks/install_helpers.yml new file mode 100644 index 0000000..edffe97 --- /dev/null +++ b/tasks/install_helpers.yml @@ -0,0 +1,12 @@ +--- +## +## Dokku config +## + +- name: Install duplicity helper script for dokku + copy: + src: files/duplicity_helper_for_dokku + dest: /usr/sbin/duplicity_helper_for_dokku + mode: "0755" + owner: root + group: root diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100644 index 0000000..673f026 --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,22 @@ +--- +# tasks file for glenux.duplicity + +# Ensure that variables are well defined +- assert: + that: + - duplicity_hostname != "" + - duplicity_work_directory != "" + - duplicity_enable != "" + +# Install required system tools +- include: prerequisites.yml + +# Install binaries & scripts +- include: install_duplicity.yml +- include: install_helpers.yml + +# Configure duplicity +- include: config_duplicity.yml +- include: config_gnupg.yml + +# diff --git a/tasks/prerequisites.yml b/tasks/prerequisites.yml new file mode 100644 index 0000000..444fcf6 --- /dev/null +++ b/tasks/prerequisites.yml @@ -0,0 +1,13 @@ +--- +## +## System prerequisistes +## +- name: Install duplicity prerequisites + apt: + name: + - librsync-dev + - gettext + - python3-pip + - gnupg + state: present +# diff --git a/templates/etc.glenux.duplicity.backends.json.j2 b/templates/etc.glenux.duplicity.backends.json.j2 new file mode 100644 index 0000000..9c5e78f --- /dev/null +++ b/templates/etc.glenux.duplicity.backends.json.j2 @@ -0,0 +1,74 @@ +[ +{% for backend in duplicity_backends %} + { + "description": "Cold storage {{backend.regionname | upper}} for {{duplicity_hostname}}", + "url": "pca://cold-archive-{{backend.regionname | lower}}-{{duplicity_hostname}}", + "env": [ + { + "name": "PCA_AUTHURL", + "value": "https://auth.cloud.ovh.net/v3" + }, + { + "name": "PCA_AUTHVERSION", + "value": "3" + }, + { + "name": "PCA_PROJECT_DOMAIN_NAME", + "value": "{{backend.project_domain_name}}" + }, + { + "name": "PCA_TENANTID", + "value": "{{backend.tenantid}}" + }, + { + "name": "PCA_USERNAME", + "value": "{{backend.username}}" + }, + { + "name": "PCA_PASSWORD", + "value": "{{backend.username}}" + }, + { + "name": "PCA_REGIONNAME", + "value": "{{backend.regionname | upper}}" + } + ], + "prefixes": ["cold_"] + }, + { + "description": "Hot storage {{backend.regionname | upper}} for {{duplicity_hostname}}", + "url": "swift://hot-archive-{{backend.regionname | lower}}-{{duplicity_hostname}}", + "env": [ + { + "name": "SWIFT_AUTHURL", + "value": "https://auth.cloud.ovh.net/v3" + }, + { + "name": "SWIFT_AUTHVERSION", + "value": "3" + }, + { + "name": "SWIFT_PROJECT_DOMAIN_NAME", + "value": "{{backend.project_domain_name}}" + }, + { + "name": "SWIFT_TENANTID", + "value": "{{backend.tenantid}}" + }, + { + "name": "SWIFT_USERNAME", + "value": "{{backend.username}}" + }, + { + "name": "SWIFT_PASSWORD", + "value": "{{backend.password}}" + }, + { + "name": "SWIFT_REGIONNAME", + "value": "{{backend.regionname |upper}}" + } + ], + "prefixes": ["hot_"] + }{% if loop.revindex0 != 0 %},{% endif %} +{% endfor %} +] diff --git a/templates/etc.glenux.duplicity.duplicity.conf.j2 b/templates/etc.glenux.duplicity.duplicity.conf.j2 new file mode 100644 index 0000000..66337e1 --- /dev/null +++ b/templates/etc.glenux.duplicity.duplicity.conf.j2 @@ -0,0 +1,3 @@ +DUPLICITY_HOSTNAME={{ duplicity_hostname }} +DUPLICITY_WORK_DIR={{ duplicity_work_directory }} +DUPLICITY_ENABLE={{ duplicity_enable | ternary('true', 'false') }} diff --git a/tests/inventory b/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/tests/test.yml b/tests/test.yml new file mode 100644 index 0000000..5201354 --- /dev/null +++ b/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - glenux.duplicity \ No newline at end of file diff --git a/vars/main.yml b/vars/main.yml new file mode 100644 index 0000000..c3dfed6 --- /dev/null +++ b/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for glenux.duplicity