Browse Source

backup: implement snapshot within enough

keep-around/828302353b6d29073527722b5a919fcee957b5e2
Loïc Dachary 1 year ago
committed by Loic Dachary
parent
commit
a62f79a41d
Signed by: dachary GPG Key ID: 992D23B392F9E4F2
  1. 4
      development-inventory/host_vars/pet-host.yml
  2. 7
      development-inventory/test-hosts.yml
  3. 71
      enough/common/openstack.py
  4. 47
      enough/internal/cli/backup.py
  5. 1
      inventory/02-all.yml
  6. 12
      molecule/backup/molecule.yml
  7. 7
      molecule/backup/playbook.yml
  8. 2
      molecule/backup/roles/backup/defaults/main.yml
  9. 20
      molecule/backup/roles/backup/tasks/main.yml
  10. 18
      molecule/backup/roles/backup/templates/prune-volume-snapshots.sh
  11. 7
      molecule/backup/roles/backup/templates/snapshot.sh.j2
  12. 7
      molecule/backup/roles/backup/templates/volume-snapshots.sh
  13. 38
      molecule/backup/tests/test_snapshot.py
  14. 2
      setup.cfg
  15. 23
      tests/enough/common/test_openstack.py
  16. 14
      tests/enough/internal/test_internal_backup.py

4
development-inventory/host_vars/pet-host.yml

@ -0,0 +1,4 @@
---
openstack_volumes:
- name: pet-volume
size: 1

7
development-inventory/test-hosts.yml

@ -18,6 +18,8 @@ all:
wekan-host:
# misc
debian-host:
# backup
pet-host:
firewall_web_server_group:
hosts:
@ -33,3 +35,8 @@ firewall_web_server_group:
wekan-group:
hosts:
wekan-host:
pets:
hosts:
# backup
pet-host:

71
enough/common/openstack.py

@ -1,4 +1,5 @@
import base64
import datetime
import json
import logging
import os
@ -239,6 +240,10 @@ class OpenStackLeftovers(Exception):
pass
class OpenStackBackupCreate(Exception):
pass
class OpenStack(OpenStackBase):
def __init__(self, config_dir, config_file):
@ -248,6 +253,13 @@ class OpenStack(OpenStackBase):
@retry(OpenStackLeftovers, tries=7)
def destroy_everything(self, prefix):
leftovers = []
for snapshot in self.o.volume.snapshot.list('--format=value', '-c', 'Name', _iter=True):
snapshot = snapshot.strip()
if prefix is None or prefix in snapshot:
leftovers.append(f'snapshot({snapshot})')
self.o.volume.snapshot.delete(snapshot)
r = self.o.stack.list('--format=json', '-c', 'Stack Name', '-c', 'Stack Status')
for name, status in [(x["Stack Name"], x["Stack Status"]) for x in json.loads(r.stdout)]:
if prefix is None or prefix in name:
@ -261,6 +273,12 @@ class OpenStack(OpenStackBase):
leftovers.append(f'image({image})')
self.o.image.delete(image)
for volume in self.o.volume.list('--format=value', '-c', 'Name', _iter=True):
volume = volume.strip()
if prefix is None or prefix in volume:
leftovers.append(f'volume({volume})')
self.o.volume.delete(volume)
for network in self.o.network.list('--format=value', '-c', 'Name', _iter=True):
network = network.strip()
if prefix is None or prefix in network:
@ -305,3 +323,56 @@ class OpenStack(OpenStackBase):
port = self.o.port.list('--server', server, '--network', network,
'--format=value', '-c', 'ID')
return port.strip() != ''
def backup_date(self):
return datetime.datetime.today().strftime('%Y-%m-%d')
def backup_create(self, volumes):
if len(volumes) == 0:
volumes = [x.strip() for x in self.o.volume.list('--format=value', '-c', 'Name')]
date = self.backup_date()
snapshots = self._backup_map()
count = 0
for volume in volumes:
s = f'{date}-{volume}'
if s not in snapshots:
self.o.volume.snapshot.create('--force', '--volume', volume, s)
count += 1
self._backup_available(volumes, date)
return count
def _backup_map(self):
return dict(self._backup_list())
def _backup_list(self):
r = self.o.volume.snapshot.list('--format=json', '-c', 'Name', '-c', 'Status',
'--limit', '5000')
return [(x["Name"], x["Status"]) for x in json.loads(r.stdout)]
@retry(OpenStackBackupCreate, tries=7)
def _backup_available(self, volumes, date):
available = []
waiting = []
for name, status in self._backup_list():
if not name.startswith(date):
continue
if status == "available":
available.append(name)
else:
waiting.append(f'{status} {name}')
available = ",".join(available)
waiting = ",".join(waiting)
progress = f'WAITING on {waiting}\nAVAILABLE {available}'
log.debug(progress)
if len(waiting) > 0:
raise OpenStackBackupCreate(progress)
def backup_prune(self, days):
before = (datetime.datetime.today() - datetime.timedelta(days)).strftime('%Y-%m-%d')
count = 0
for name, status in self._backup_list():
if name[:10] > before:
continue
self.o.volume.snapshot.delete(name)
count += 1
return count

47
enough/internal/cli/backup.py

@ -0,0 +1,47 @@
import logging
from cliff.command import Command
from enough import settings
from enough.common.openstack import OpenStack
from enough.common import dotenough
from enough.cli import openstack
class Create(Command):
"Create backups."
log = logging.getLogger(__name__)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.add_argument('volumes', nargs='*')
openstack.set_common_options(parser)
return parser
def take_action(self, parsed_args):
args = vars(self.app.options)
args.update(vars(parsed_args))
d = dotenough.DotEnoughOpenStack(settings.CONFIG_DIR, args['domain'])
d.ensure()
o = OpenStack(settings.CONFIG_DIR, args['clouds'])
o.backup_create(args['volumes'])
class Prune(Command):
"Prune old backups."
log = logging.getLogger(__name__)
def get_parser(self, prog_name):
parser = super().get_parser(prog_name)
parser.add_argument('days', type=int, default=30)
openstack.set_common_options(parser)
return parser
def take_action(self, parsed_args):
args = vars(self.app.options)
args.update(vars(parsed_args))
d = dotenough.DotEnoughOpenStack(settings.CONFIG_DIR, args['domain'])
d.ensure()
o = OpenStack(settings.CONFIG_DIR, args['clouds'])
o.backup_prune(args['days'])

1
inventory/02-all.yml

@ -52,4 +52,5 @@ openvpn-group:
enough-user-group:
children:
backup-group:
api-group:

12
molecule/backup/molecule.yml

@ -5,18 +5,18 @@ lint:
name: yamllint
platforms:
- name: bind-host
- name: weblate-host
groups:
- pets
- name: icinga-host
- name: packages-host
- name: pet-host
provisioner:
name: ansible
options:
i: ../../development-inventory
limit: bind-host,weblate-host,localhost
limit: bind-host,icinga-host,packages-host,pet-host,localhost
lint:
name: ansible-lint
env:
ANSIBLE_ROLES_PATH: roles:../infrastructure/roles:../firewall/roles:../backup/roles
ANSIBLE_ROLES_PATH: roles:../infrastructure/roles:../firewall/roles:../bind/roles:../icinga/roles:../enough-nginx/roles:../certificate/roles:../jdauphant.nginx/roles
inventory:
links:
group_vars: ../../inventory/group_vars
@ -34,6 +34,6 @@ verifier:
options:
v: True
s: True
# k: test_snapshots
k: test_snapshots
lint:
name: flake8

7
molecule/backup/playbook.yml

@ -1,4 +1,11 @@
---
- import_playbook: ../infrastructure/buster-playbook.yml
- import_playbook: ../firewall/firewall-playbook.yml
- import_playbook: ../icinga/test-icinga-playbook.yml
- import_playbook: ../bind/bind-playbook.yml
- import_playbook: ../bind/bind-client-playbook.yml
- import_playbook: ../icinga/icinga-playbook.yml
- import_playbook: ../packages/packages-playbook.yml
- import_playbook: ../packages/enough-pip-playbook.yml
- import_playbook: backup-playbook.yml
- import_playbook: ../misc/uninstall-ntp.yml

2
molecule/backup/roles/backup/defaults/main.yml

@ -0,0 +1,2 @@
---
backup_retention_days: 30

20
molecule/backup/roles/backup/tasks/main.yml

@ -16,7 +16,6 @@
mode: +x
loop:
- backup
- volume-snapshots
- name: copy prune scripts
template:
@ -25,8 +24,25 @@
mode: +x
loop:
- prune-backup
- prune-volume-snapshots
- name: apt-get install python-openstackclient
apt:
name: python-openstackclient
- name: mkdir -p ~/.enough/{{ domain }}
file:
state: directory
path: "/root/.enough/{{ domain }}"
- name: cp openrc.sh ~/.enough/{{ domain }}/openrc.sh
template:
src: openrc.sh
dest: "/root/.enough/{{ domain }}/openrc.sh"
- name: copy snapshot backup
template:
src: "{{ item }}.j2"
dest: "/etc/cron.daily/{{ item }}"
mode: +x
loop:
- snapshot.sh

18
molecule/backup/roles/backup/templates/prune-volume-snapshots.sh

@ -1,18 +0,0 @@
#!/bin/bash
source /usr/lib/backup/openrc.sh
numdays=${1:-30}
stamp1=`date --date "$numdays days ago" '+%s'`
regex="^([0-9]{4}-[0-9]{2}-[0-9]{2}).*$"
openstack ${OS_INSECURE} volume snapshot list -f value -c Name --limit 10000 |
while read snapshot
do
if [[ $snapshot =~ $regex ]] ; then
snapshotdate=`echo $snapshot | sed -r "s/$regex/\1/"`
stamp2=`date --date "$snapshotdate" '+%s'`
if [[ $stamp1 -ge $stamp2 ]] ; then
openstack ${OS_INSECURE} volume snapshot delete $snapshot
fi
fi
done

7
molecule/backup/roles/backup/templates/snapshot.sh.j2

@ -0,0 +1,7 @@
#!/bin/bash
set -ex
days=${1:-{{ backup_retention_days }}}
docker run -v $HOME/.enough:/root/.enough --rm enoughcommunity/enough:latest --domain {{ domain }} backup create
docker run -v $HOME/.enough:/root/.enough --rm enoughcommunity/enough:latest --domain {{ domain }} backup prune $days

7
molecule/backup/roles/backup/templates/volume-snapshots.sh

@ -1,7 +0,0 @@
#!/bin/bash
source /usr/lib/backup/openrc.sh
for volume in $(openstack ${OS_INSECURE} volume list --column Name --format value) ; do
openstack ${OS_INSECURE} volume snapshot create --force --volume $volume $(date +%Y-%m-%d)-$volume
done

38
molecule/backup/tests/test_snapshot.py

@ -4,7 +4,7 @@ testinfra_hosts = ['bind-host']
def openstack(host, cmd):
cmd = host.run(f"""
. /usr/lib/backup/openrc.sh
openstack $OS_INSECURE {cmd}
openstack {cmd}
""")
print(cmd.stderr)
assert 0 == cmd.rc
@ -13,42 +13,20 @@ def openstack(host, cmd):
def expected_snapshots(host, count):
assert count == openstack(
host, "volume snapshot list -f value -c Name | grep -c test-backup-volume")
host, "volume snapshot list -f value -c Name | grep -c pet-volume")
def test_snapshots(host):
# we need --insecure during tests otherwise going back in time a few days
# may invalidate some certificates and result in errors such as:
# SSL exception connecting to
# https://auth.cloud.ovh.net/v2.0/tokens: [SSL: CERTIFICATE_VERIFY_FAILED]
with host.sudo():
cmd = host.run("echo export OS_INSECURE=--insecure >> /usr/lib/backup/openrc.sh")
cmd = host.run("/etc/cron.daily/snapshot.sh")
print(cmd.stdout)
print(cmd.stderr)
assert 0 == cmd.rc
openstack(host, "volume create --size 1 test-backup-volume")
cmd = host.run("/etc/cron.daily/prune-volume-snapshots 0")
print(cmd.stdout)
print(cmd.stderr)
assert 0 == cmd.rc
try:
with host.sudo():
host.run("timedatectl set-ntp 0")
cmd = host.run("""
set -x
date -s '-15 days'
bash -x /etc/cron.daily/volume-snapshots
date -s '-30 days'
bash -x /etc/cron.daily/volume-snapshots
""")
host.run("timedatectl set-ntp 1")
expected_snapshots(host, '1')
cmd = host.run("/etc/cron.daily/snapshot.sh 0")
print(cmd.stdout)
print(cmd.stderr)
assert 0 == cmd.rc
expected_snapshots(host, '2')
host.run("bash -x /etc/cron.daily/prune-volume-snapshots 30")
expected_snapshots(host, '1')
finally:
host.run("timedatectl set-ntp 1")
host.run("bash -x /etc/cron.daily/prune-volume-snapshots 0")
openstack(host, "volume delete test-backup-volume || true")
assert openstack(host, "snapshot list -f value -c Name") == ""

2
setup.cfg

@ -79,6 +79,8 @@ enough.internal.cli =
build_image = enough.internal.cli.docker:Build
create_service = enough.internal.cli.docker:Create
install = enough.internal.cli.install:InstallScript
backup_create = enough.internal.cli.backup:Create
backup_prune = enough.internal.cli.backup:Prune
create_test_subdomain = enough.internal.cli.test:CreateTestSubdomain
[build_sphinx]

23
tests/enough/common/test_openstack.py

@ -148,3 +148,26 @@ def test_network(openstack_name):
o.network_and_subnet_create(openstack_name, '10.11.12.0/24')
assert o.network_exists(openstack_name)
assert o.subnet_exists(openstack_name)
@pytest.mark.skipif('SKIP_OPENSTACK_INTEGRATION_TESTS' in os.environ,
reason='skip integration test')
def test_backup_create_with_name(openstack_name, caplog):
o = OpenStack(settings.CONFIG_DIR, 'inventory/group_vars/all/clouds.yml')
o.o.volume.create('--size=1', openstack_name)
assert o.backup_create([openstack_name]) == 1
assert o.backup_create([openstack_name]) == 0
available_snapshot = f'AVAILABLE {o.backup_date()}-{openstack_name}'
assert available_snapshot in caplog.text
assert o.backup_prune(0) == 1
assert o.backup_prune(0) == 0
@pytest.mark.skipif('SKIP_OPENSTACK_INTEGRATION_TESTS' in os.environ,
reason='skip integration test')
def test_backup_create_no_names(openstack_name, caplog):
o = OpenStack(settings.CONFIG_DIR, 'inventory/group_vars/all/clouds.yml')
o.o.volume.create('--size=1', openstack_name)
o.backup_create([])
available_snapshot = f'AVAILABLE {o.backup_date()}-{openstack_name}'
assert available_snapshot in caplog.text

14
tests/enough/internal/test_internal_backup.py

@ -0,0 +1,14 @@
from tests.modified_environ import modified_environ
from enough.internal import cmd
def test_enough_backup(capsys, mocker):
# do not tamper with logging streams to avoid
# ValueError: I/O operation on closed file.
mocker.patch('cliff.app.App.configure_logging')
mocker.patch('enough.common.openstack.OpenStack.backup_create',
side_effect=lambda *args, **kwargs: print('BACKUP'))
with modified_environ(OS_CLIENT_CONFIG_FILE="/dev/null"):
assert cmd.main(['backup', 'create']) == 0
out, err = capsys.readouterr()
assert 'BACKUP' in out
Loading…
Cancel
Save