*untested* Revamp of backup protocol and some cleanup

inventories/core_infrastructure.yml

@@ -7,17 +7,26 @@ all:
     web:
       ansible_host: "10.26.48.3"
       docker_backup:
-        - container_name: http-test
+        - container_name: gitea_new
           directories_to_backup:
-            - /var/www/html
+            - /gitea_new
-          backup_dir: /backup/httptest
+          backup_dir: /backup/gitea_new
-          backup_name_prefix: http-test
+          backup_name_prefix: gitea_new
           max_backups_kept: 3
-        - container_name: gitea
+          mysql_backup:
-          directories_to_backup:
+            target: 10.26.48.3
-            - /gitea
+            target_port: 3306
-          backup_dir: /backup/gitea
+            target_username: "ansible"
-          backup_name_prefix: gitea
+            target password: !vault |
+              $ANSIBLE_VAULT;1.1;AES256
+              35636232613339633862326664653765616631623032663465326631323030323536663539313336
+              3462336531356362323636323439363465343465613131370a353632396338386135336133623739
+              63643933663937643333396237623935313265393065653465363133386461643435316337373430
+              6531663233633931390a313265333031346430653730353366393832333133653532666631623661
+              66316463663333623639623465383964616336656634313366656632643032326439
+            database_name: gitea
+            backup_dir: /backup/gitea_new_db
+            backup_name_prefix: gitea_new_db
             max_backups_kept: 3
         - container_name: infrastructure-compose-web_recipes-1
           directories_to_backup:
@@ -25,16 +34,8 @@ all:
           backup_dir: /backup/tandoor
           backup_name_prefix: tandoor
           max_backups_kept: 3
-        - container_name: nextcloud
-          directories_to_backup:
-            - /nextcloud
-          backup_dir: /backup/nextcloud
-          backup_name_prefix: nextcloud
-          max_backups_kept: 3
     database:
       ansible_host: "10.12.34.3"
-    publicworks:
-      ansible_host: "10.77.7.3"
     dmz:
       ansible_host: "172.16.132.4"
       docker_backup:
@@ -112,34 +113,6 @@ all:
           max_backups_kept: 3
     nfsserver:
       ansible_host: "10.42.0.3"
-    openocean:
-      ansible_host: "172.16.132.2"
-      backup_url: "172.16.132.2:1234"
-      backup_user: "backup"
-      backup_user_password: !vault |
-        $ANSIBLE_VAULT;1.1;AES256
-        31336636333061393333326263353835636431313739613462356531623532663137626466613433
-        3438616239333536643835643933333461323666343864640a386361306163623261656630643837
-        65396139333264636333656337383766393931393934373335646231336330656561303039326665
-        3331636339356236330a333330373264306163393162386330393763613332376261373433303035
-        65633034366439343232356639346665326234666630633063616437376131396463
-      backup_location: "/backup"
-      backup_number_to_keep: 10
-    boardwalk:
-      ansible_host: "10.77.7.2"
-      backup_url: "10.77.7.2:1234"
-      backup_user: "backup"
-      backup_user_password: !vault |
-        $ANSIBLE_VAULT;1.1;AES256
-        34313763623733323133393734326533333661343239393037666462323732393839386131393165
-        6233376533623431333238373039353330616265363566320a393730343938306430313864666534
-        36363736303436643163313636303931323032366136616634366363383036303737356336343638
-        6566663062336566320a323536303233393431363263313933643839303435356266656136343438
-        35306638356564333962656433323735656136386130373233393765616265306636
-      backup_location: "/backup"
-      backup_number_to_keep: 10
-    monitor:
-      ansible_host: "10.42.0.4"
     nextcloud:
       ansible_host: "10.26.48.4"
   children:
@@ -147,14 +120,9 @@ all:
       hosts:
         web:
         database:
-        publicworks:
         dmz:
         games:
         nextcloud:
-    pfsense_nodes:
-      hosts:
-        openocean:
-        boardwalk:
     database_nodes:
       database:
     

playbooks/backup_protocol.yml

@@ -1,141 +1,13 @@
 ---
-# Bits an pieces of this play are stolen from https://github.com/gavinwill/ansible-role-pfsense-backup/tree/main
-- name: PFSense Backups
-  hosts: pfsense_nodes
-  gather_facts: false
-  collections:
-    - ansible.posix
-  tasks:
-    - name: "Log first stage start time"
-      ansible.builtin.debug:
-        msg: "{{ now() }}"
-      run_once: true
-      delegate_to: blacktide
-
-    - name: Mount backup share
-      ansible.posix.mount:
-        path: /backup
-        src: "10.42.0.3:/backups/pfsense"
-        fstype: nfs
-        opts: "hard,intr,nodev,nosuid,noexec"
-        state: ephemeral
-      delegate_to: blacktide
-      run_once: true
-      become: true
-      become_method: sudo
-
-    - name: Get Cookies and CSRF Token
-      ansible.builtin.uri:
-        url: "http://{{ backup_url }}/diag_backup.php"
-        validate_certs: false
-        method: GET
-        return_content: true
-      register: pfsense_cookie_token
-      no_log: "{{ pfsense_backup_disable_logging | default(false) }}"
-      delegate_to: blacktide
-
-    - name: Set CSRF Token and Cookie Fact
-      ansible.builtin.set_fact:
-        pfsense_backup_csrf: "{{ pfsense_cookie_token.content | regex_search('var\\s+csrfMagicToken\\s+=\\s+\\\"([a-f0-9sidp:;,]+)\\\"', '\\1') }}"
-        pfsense_backup_cookie: "{{ pfsense_cookie_token.set_cookie }}"
-      no_log: "{{ pfsense_backup_disable_logging | default(false) }}"
-
-    - name: Authenticate with backup page and register backup CSRF
-      ansible.builtin.uri:
-        url: "http://{{ backup_url }}/diag_backup.php"
-        validate_certs: false
-        follow_redirects: false
-        method: POST
-        return_content: true
-        body_format: form-urlencoded
-        status_code: 302
-        body:
-          login: Login
-          usernamefld: "{{ backup_user }}"
-          passwordfld: "{{ backup_user_password }}"
-          __csrf_magic: "{{ pfsense_backup_csrf }}"
-        headers:
-          Cookie: "{{ pfsense_backup_cookie }}"
-      register: pfsense_cookie_token_2
-      delegate_to: blacktide
-      no_log: "{{ pfsense_backup_disable_logging | default(false) }}"
-
-    - name: Set Cookie Fact from Backup page
-      ansible.builtin.set_fact:
-        pfsense_backup_cookie_1: "{{ pfsense_cookie_token_2.set_cookie }}"
-      no_log: "{{ pfsense_backup_disable_logging | default(false) }}"
-
-    - name: Fetch Target page for new CSRF token
-      ansible.builtin.uri:
-        url: "http://{{ backup_url }}/diag_backup.php"
-        validate_certs: false
-        follow_redirects: false
-        method: GET
-        return_content: true
-        headers:
-          Cookie: "{{ pfsense_backup_cookie_1 }}"
-      register: pfsense_cookie_token_3
-      delegate_to: blacktide
-      no_log: "{{ pfsense_backup_disable_logging | default(false) }}"
-
-    - name: Set fact for CSRF Token and Cookie
-      ansible.builtin.set_fact:
-        pfsense_backup_csrf_1: "{{ pfsense_cookie_token_3.content | regex_search('var\\s+csrfMagicToken\\s+=\\s+\\\"([a-f0-9sidp:;,]+)\\\"', '\\1') }}"
-        pfsense_backup_cookie_2: "{{ pfsense_cookie_token_3.set_cookie }}"
-      no_log: "{{ pfsense_backup_disable_logging | default(false) }}"
-
-    - name: Download Backup Configuration
-      ansible.builtin.uri:
-        url: "http://{{ backup_url }}/diag_backup.php"
-        validate_certs: false
-        follow_redirects: false
-        method: "POST"
-        return_content: true
-        body_format: form-urlencoded
-        body:
-          download: download
-          backupssh: "yes"
-          backupdata: "yes"
-          donotbackuprrd: "yes"
-          __csrf_magic: "{{ pfsense_backup_csrf_1 }}"
-        headers:
-          Cookie: "{{ pfsense_backup_cookie_2 }}"
-        dest: "{{ backup_location }}/{{ inventory_hostname }}_{{ now().strftime('%Y%m%d%H%M%S') }}.xml"
-      changed_when: false
-      delegate_to: blacktide
-      no_log: "{{ pfsense_backup_disable_logging | default(false) }}"
-
-    - name: Find all PFSense backups for the current host
-      ansible.builtin.find:
-        paths: "{{ backup_location }}"
-        patterns: "{{ inventory_hostname }}*"
-      register: all_pfsense_backups
-      delegate_to: blacktide
-
-    - name: If too many backups kept
-      when: all_pfsense_backups.files | length > backup_number_to_keep
-      block:
-        - name: Get the oldest file paths
-          ansible.builtin.set_fact:
-            oldest_file_paths: >-
-              {{ (all_pfsense_backups.files | sort(attribute='mtime'))[:all_pfsense_backups.files | length - backup_number_to_keep] |
-                  map(attribute='path') | list }}
-          
-        - name: Remove the files
-          ansible.builtin.file:
-            path: "{{ item }}"
-            state: absent
-          loop: "{{ oldest_file_paths }}"
-          delegate_to: blacktide
-    
 - name: Docker Infrastructure Compose Backups
-  hosts: docker_nodes,!publicworks
+  hosts: docker_nodes
   become: true
   become_method: sudo
   gather_facts: false
   collections:
     - community.docker
     - community.general
+    - ansible.mysql
   vars:
     env_backups_to_keep: 10
   tasks:
@@ -154,7 +26,7 @@
       ansible.builtin.include_role:
         name: docker_backup
       vars:
-        backup_rules: "{{ item }}"
+        docker_backup_rules: "{{ item }}"
       when: docker_backup is defined and docker_backup | length != 0
       loop: "{{ docker_backup }}"
 

playbooks/roles/docker_backup/defaults/main.yml

@@ -1,2 +1,2 @@
 ---
-backup_rules: ""
+docker_backup_rules: ""

playbooks/roles/docker_backup/tasks/main.yml

@@ -2,7 +2,7 @@
 - name: Ensure backup rules contains the necessary fields
   ansible.builtin.fail:
     msg: "You must define {{ backup_rule_checker }} in the backup ruleset"
-  when: not backup_rules[backup_rule_checker] is defined
+  when: not docker_backup_rules[backup_rule_checker] is defined
   loop:
     - container_name
     - directories_to_backup
@@ -14,42 +14,49 @@
 
 - name: Make sure backup dir exists
   ansible.builtin.file:
-    path: "{{ backup_rules['backup_dir'] }}"
+    path: "{{ docker_backup_rules['backup_dir'] }}"
     state: directory
     recurse: true
 
 - name: Stop the running container
   community.docker.docker_container:
-    name: "{{ backup_rules.container_name }}"
+    name: "{{ docker_backup_rules.container_name }}"
     state: stopped
 
 - name: Archive necessary directories
   community.general.archive:
-    path: "{{ backup_rules.directories_to_backup }}"
+    path: "{{ docker_backup_rules.directories_to_backup }}"
     dest: >-
-      {{ backup_rules.backup_dir }}/{{ backup_rules.backup_name_prefix }}_{{ now().strftime("%Y%m%d%H%M%S") }}.tar.gz
+      {{ docker_backup_rules.backup_dir }}/{{ docker_backup_rules.backup_name_prefix }}_{{ now().strftime("%Y%m%d%H%M%S") }}.tar.gz
     format: gz
   async: 3600
   poll: 60
 
+- name: Take a MySQL DB backup if necessary
+  ansible.builtin.import_role:
+    name: mysql_backup
+  vars:
+    mysql_backup_rules: "{{ docker_backup_rules['mysql_backup'] }}"
+  when: docker_backup_rules['mysql_backup'] is defined
+  
 - name: Start the stopped container
   community.docker.docker_container:
-    name: "{{ backup_rules.container_name }}"
+    name: "{{ docker_backup_rules.container_name }}"
     state: started
 
 - name: Find all files that start with the backup_name_prefix
   ansible.builtin.find:
-    paths: "{{ backup_rules.backup_dir }}"
+    paths: "{{ docker_backup_rules.backup_dir }}"
-    patterns: "{{ backup_rules.backup_name_prefix }}*"
+    patterns: "{{ docker_backup_rules.backup_name_prefix }}*"
   register: all_backup_files
 
 - name: If too many backups kept
-  when: all_backup_files.files | length > backup_rules.max_backups_kept
+  when: all_backup_files.files | length > docker_backup_rules.max_backups_kept
   block:
     - name: Get the oldest file paths
       ansible.builtin.set_fact:
         oldest_file_paths: >-
-          {{ (all_backup_files.files | sort(attribute='mtime'))[:all_backup_files.files | length - backup_rules.max_backups_kept] |
+          {{ (all_backup_files.files | sort(attribute='mtime'))[:all_backup_files.files | length - docker_backup_rules.max_backups_kept] |
               map(attribute='path') | list }}
       
     - name: Remove the files

playbooks/roles/mysql_backup/README.md

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

playbooks/roles/mysql_backup/defaults/main.yml

@@ -0,0 +1,2 @@
+---
+mysql_backup_rules: ""

playbooks/roles/mysql_backup/meta/main.yml

@@ -0,0 +1,52 @@
+galaxy_info:
+  author: your name
+  description: your role description
+  company: your company (optional)
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+
+  # Choose a valid license ID from https://spdx.org - some suggested licenses:
+  # - BSD-3-Clause (default)
+  # - MIT
+  # - GPL-2.0-or-later
+  # - GPL-3.0-only
+  # - Apache-2.0
+  # - CC-BY-4.0
+  license: license (GPL-2.0-or-later, MIT, etc)
+
+  min_ansible_version: 2.1
+
+  # If this a Container Enabled role, provide the minimum Ansible Container version.
+  # min_ansible_container_version:
+
+  #
+  # Provide a list of supported platforms, and for each platform a list of versions.
+  # If you don't wish to enumerate all versions for a particular platform, use 'all'.
+  # To view available platforms and versions (or releases), visit:
+  # https://galaxy.ansible.com/api/v1/platforms/
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is a keyword that describes
+    # and categorizes the role. Users find roles by searching for tags. Be sure to
+    # remove the '[]' above, if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of alphanumeric characters.
+    #       Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line. Be sure to remove the '[]' above,
+  # if you add dependencies to this list.

playbooks/roles/mysql_backup/tasks/main.yml

@@ -0,0 +1,64 @@
+---
+- name: Ensure backup rules contains the necessary fields
+  ansible.builtin.fail:
+    msg: "You must define {{ backup_rule_checker }} in the backup ruleset"
+  when: not mysql_backup_rules[backup_rule_checker] is defined
+  loop:
+    - target
+    - target_port
+    - target_username
+    - target_password
+    - database_name
+    - backup_dir
+    - backup_name_prefix
+    - max_backups_kept
+  loop_control:
+    loop_var: backup_rule_checker
+
+- name: Ensure necessary packages are installed
+  ansible.builtin.package:
+    name:
+      - "python3-PyMySQL"
+      - "mysql_community_client"
+    state: present
+
+- name: Make sure backup dir exists
+  ansible.builtin.file:
+    path: "{{ mysql_backup_rules['backup_dir'] }}"
+    state: directory
+    recurse: true
+
+- name: Take a backup of the specified database
+  ansible.mysql.mysql_db:
+    name: "{{ mysql_backup_rules['database_name'] }}"
+    state: dump
+    target: "{{ mysql_backup_rules['backup_dir'] }}/{{ mysql_backup_rules['backup_name_prefix'] }}_{{ now().strftime('%Y%m%d%H%M%S') }}.sql.gz"
+    login_host: "{{ mysql_backup_rules['target'] }}"
+    login_port: "{{ mysql_backup_rules['target_port'] }}"
+    login_user: "{{ mysql_backup_rules['target_username'] }}"
+    login_password: "{{ mysql_backup_rules['target_password'] }}"
+
+- name: Find all files that start with the backup_name_prefix
+  ansible.builtin.find:
+    paths: "{{ mysql_backup_rules.backup_dir }}"
+    patterns: "{{ mysql_backup_rules.backup_name_prefix }}*"
+  register: all_backup_files
+
+- name: If too many backups kept
+  when: all_backup_files.files | length > mysql_backup_rules.max_backups_kept
+  block:
+    - name: Get the oldest file paths
+      ansible.builtin.set_fact:
+        oldest_file_paths: >-
+          {{ (all_backup_files.files | sort(attribute='mtime'))[:all_backup_files.files | length - mysql_backup_rules.max_backups_kept] |
+              map(attribute='path') | list }}
+      
+    - name: Remove the files
+      ansible.builtin.file:
+        path: "{{ old_file_to_delete }}"
+        state: absent
+      loop: "{{ oldest_file_paths }}"
+      loop_control:
+        loop_var: old_file_to_delete
+
+