Browse Source

api: implement the bind endpoint

keep-around/968965ffd776479a127508955201629b3737a560
singuliere 2 years ago
parent
commit
4e103abbaf
No known key found for this signature in database GPG Key ID: 900857755EF189C2
  1. 2
      ansible.cfg
  2. 2
      enough/api/admin.py
  3. 25
      enough/api/data/docker-compose-api.yml
  4. 10
      enough/api/data/enough.service
  5. 2
      enough/api/models.py
  6. 3
      enough/api/tests.py
  7. 42
      enough/api/views.py
  8. 8
      enough/cli/manage.py
  9. 5
      enough/cmd.py
  10. 5
      enough/common/data/base.dockerfile
  11. 13
      enough/internal/data/enough-source.dockerfile
  12. 10
      enough/internal/data/enough.dockerfile
  13. 5
      enough/settings.py
  14. 2
      enough/urls.py
  15. 1
      inventories/common/02-all.yml
  16. 1
      inventories/common/all.yml
  17. 2
      inventories/common/firewall.yml
  18. 2
      inventories/common/host_vars/api-host/api.yml
  19. 14
      molecule/api/api-playbook.yml
  20. 1
      molecule/api/create.yml
  21. 1
      molecule/api/destroy.yml
  22. 44
      molecule/api/molecule.yml
  23. 13
      molecule/api/playbook.yml
  24. 2
      molecule/api/roles/api/defaults/main.yml
  25. 3
      molecule/api/roles/api/files/apt-preferences.d/default
  26. 3
      molecule/api/roles/api/files/apt-preferences.d/python3
  27. 151
      molecule/api/roles/api/tasks/api.yml
  28. 2
      molecule/api/roles/api/tasks/main.yml
  29. 35
      molecule/api/tests/test_api.py
  30. 55
      molecule/packages/roles/enough-pip/tasks/enough-pip.yml
  31. 2
      molecule/preprod/molecule.yml
  32. 39
      tests/enough/api/test_api.py
  33. 4
      tests/enough/common/test_common_docker.py
  34. 4
      tox.ini

2
ansible.cfg

@ -1,2 +1,2 @@
[defaults]
roles_path = molecule/infrastructure/roles:molecule/authorized_keys/roles:molecule/backup/roles:molecule/bind/roles:molecule/icinga/roles:molecule/postfix/roles:molecule/weblate/roles:molecule/packages/roles:molecule/jdauphant.nginx/roles:molecule/letsencrypt-nginx/roles:molecule/wazuh/roles:molecule/firewall/roles
roles_path = molecule/infrastructure/roles:molecule/authorized_keys/roles:molecule/backup/roles:molecule/bind/roles:molecule/icinga/roles:molecule/postfix/roles:molecule/weblate/roles:molecule/packages/roles:molecule/jdauphant.nginx/roles:molecule/letsencrypt-nginx/roles:molecule/wazuh/roles:molecule/firewall/roles:molecule/api/roles

2
enough/api/admin.py

@ -1,3 +1,3 @@
from django.contrib import admin
# from django.contrib import admin
# Register your models here.

25
enough/api/data/docker-compose-api.yml

@ -0,0 +1,25 @@
version: "3"
services:
api:
command: /sbin/init
image: enough:latest
working_dir: /opt
tmpfs:
- /tmp
- /run
- /run/lock
volumes:
- /sys/fs/cgroup:/sys/fs/cgroup:ro
- /opt/.enough:/root/.enough
security_opt:
- seccomp=unconfined
cap_add:
- IPC_LOCK
ports:
- "8000:8000"
networks:
- enough
networks:
enough:

10
enough/api/data/enough.service

@ -0,0 +1,10 @@
[Unit]
Description=Enough API
Requires=network-online.target
After=network-online.target
[Service]
ExecStart=env PATH=/opt/venv/bin:${PATH} enough manage runserver 0.0.0.0:8000
[Install]
WantedBy=multi-user.target

2
enough/api/models.py

@ -1,3 +1,3 @@
from django.db import models
# from django.db import models
# Create your models here.

3
enough/api/tests.py

@ -1,3 +0,0 @@
from django.test import TestCase
# Create your tests here.

42
enough/api/views.py

@ -1,8 +1,40 @@
from django.views.decorators.csrf import csrf_exempt
from django.http import JsonResponse
from rest_framework.decorators import api_view
from io import StringIO
import sh
from enough import configuration
import re
import json
@csrf_exempt
def run_ansible(*args, **kwargs):
out = StringIO()
kwargs['_out'] = out
sh.ansible(*args, **kwargs)
return out.getvalue()
@api_view(['POST'])
def bind(request):
return JsonResponse({
'a': 'b',
})
confdir = configuration.get_directory(None)
bind_host = request.data.get('bind_host', 'bind-host')
args = ['server=localhost']
for k in ('zone', 'record', 'ttl', 'type', 'value'):
if k in request.data:
args.append(f'{k}={request.data[k]}')
r = run_ansible('-i', f'{bind_host},',
'--private-key', f'{confdir}/id_rsa',
'--user=debian',
bind_host,
'--one-line',
f'--playbook-dir={confdir}',
'-m', 'nsupdate', '-a', " ".join(args),
_env={
'ANSIBLE_NOCOLOR': 'true',
'ANSIBLE_HOST_KEY_CHECKING': 'False',
})
json_result = re.sub(r'.*?=> ', '', r)
print(json_result)
result = json.loads(json_result)
return JsonResponse({"out": result}, status=201)

8
enough/cli/manage.py

@ -1,6 +1,7 @@
from enough import configuration
from cliff.command import Command
import argparse
import os
import sys
class Manage(Command):
@ -8,9 +9,12 @@ class Manage(Command):
def get_parser(self, prog_name):
parser = super(Manage, self).get_parser(prog_name)
parser.add_argument('args', nargs=argparse.REMAINDER)
return parser
def take_action(self, parsed_args):
d = configuration.get_directory(self.app.options.domain)
os.environ.setdefault('ENOUGH_BASE_DIR', d)
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'enough.settings')
try:
from django.core.management import execute_from_command_line
@ -20,4 +24,4 @@ class Manage(Command):
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(['enough', 'runserver'])
execute_from_command_line(['enough', *parsed_args.args])

5
enough/cmd.py

@ -16,6 +16,11 @@ class EnoughApp(App):
deferred_help=True,
)
def build_option_parser(self, description, version, argparse_kwargs=None):
parser = super().build_option_parser(description, version, argparse_kwargs)
parser.add_argument('--domain', help='Enough domain name')
return parser
def main(argv=sys.argv[1:]):
myapp = EnoughApp()

5
enough/common/data/base.dockerfile

@ -1,8 +1,9 @@
FROM debian:buster
RUN apt-get update && \
apt-get install --quiet -y curl virtualenv python3 gcc libffi-dev libssl-dev python3-dev make \
systemd systemd-sysv
apt-get install --quiet -y curl virtualenv python3 gcc libffi-dev libssl-dev python3-dev make git \
systemd systemd-sysv \
openssh-server
RUN curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh
RUN curl -L "https://github.com/docker/compose/releases/download/1.24.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose && chmod +x /usr/local/bin/docker-compose

13
enough/internal/data/enough-source.dockerfile

@ -0,0 +1,13 @@
ARG IMAGE_NAME
FROM ${IMAGE_NAME}
COPY requirements.txt /tmp
RUN pip3 install -r /tmp/requirements.txt
COPY dist/* .
RUN pip3 install *.tar.gz
RUN python -m enough.internal.cmd install --service > /etc/systemd/system/enough.service && systemctl enable enough
CMD [ "help" ]
ENTRYPOINT [ "python", "-m", "enough.internal.cmd" ]

10
enough/internal/data/enough.dockerfile

@ -1,9 +1,9 @@
ARG IMAGE_NAME
FROM ${IMAGE_NAME}
ARG ENOUGH_VERSION
ARG PIP3_OPTS
COPY dist/* .
RUN pip3 install ${PIP3_OPTS} enough==${ENOUGH_VERSION} # install the package and all dependencies
RUN pip3 install --no-cache-dir --force-reinstall --no-deps ${PIP3_OPTS} enough==${ENOUGH_VERSION} # replace this comment with timestamp to force reinstallation of the packages even if the version does not change
RUN pip install *.tar.gz
RUN python -m enough.internal.cmd install --service > /etc/systemd/system/enough.service && systemctl enable enough
CMD [ "--help" ]
ENTRYPOINT [ "python", "-m", "enough.internal.cmd" ]

5
enough/settings.py

@ -13,8 +13,7 @@ https://docs.djangoproject.com/en/2.1/ref/settings/
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BASE_DIR = os.environ.get('ENOUGH_BASE_DIR', 'enough.community')
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.1/howto/deployment/checklist/
@ -25,7 +24,7 @@ SECRET_KEY = 'mhh^de)&ky$ke$@z#7^zd+4&g083tldl@d7km-6)jc=cygb+07'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
ALLOWED_HOSTS = ['*']
# Application definition

2
enough/urls.py

@ -19,5 +19,5 @@ from enough.api import views
# Wire up our API using automatic URL routing.
# Additionally, we include login URLs for the browsable API.
urlpatterns = [
path('bind/', views.bind),
path('bind/', views.bind, name='bind'),
]

1
inventories/common/02-all.yml

@ -17,6 +17,7 @@ enough:
wazuh_agent:
hosts:
api-host:
bind-host:
chat-host:
forum-host:

1
inventories/common/all.yml

@ -1,6 +1,7 @@
---
all:
hosts:
api-host:
bind-host:
chat-host:
forum-host:

2
inventories/common/firewall.yml

@ -1,6 +1,7 @@
# ssh
firewall_ssh_server_group:
hosts:
api-host:
bind-host:
postfix-host:
icinga-host:
@ -48,6 +49,7 @@ firewall_wazuh_client_group:
# web
firewall_web_server_group:
hosts:
api-host:
chat-host:
forum-host:
gitlab-host:

2
inventories/common/host_vars/api-host/api.yml

@ -0,0 +1,2 @@
---
api_vhost_fqdn: api.{{ domain }}

14
molecule/api/api-playbook.yml

@ -0,0 +1,14 @@
---
- name: install API
hosts: api-host
become: true
roles:
- role: ansible-role-docker
- role: docker
- role: api
- role: letsencrypt-nginx
vars:
letsencrypt_nginx_reverse_proxy: 127.0.0.1:8000
letsencrypt_nginx_fqdn: "{{ api_vhost_fqdn }}"

1
molecule/api/create.yml

@ -0,0 +1 @@
../infrastructure/create.yml

1
molecule/api/destroy.yml

@ -0,0 +1 @@
../infrastructure/destroy.yml

44
molecule/api/molecule.yml

@ -0,0 +1,44 @@
---
driver:
name: openstack
lint:
name: yamllint
platforms:
- name: bind-host
flavor: "s1-2"
- name: api-host
flavor: "s1-2"
- name: packages-host
flavor: "s1-4"
- name: icinga-host
flavor: "s1-2"
- name: gitlab-host
flavor: "s1-4"
provisioner:
name: ansible
options:
i: ../../inventories/common/firewall.yml
limit: bind-host,packages-host,icinga-host,api-host,gitlab-host,localhost
lint:
name: ansible-lint
env:
ANSIBLE_ROLES_PATH: roles:../infrastructure/roles:../firewall/roles:../bind/roles:../packages/roles:../gitlab/roles:../icinga/roles:../jdauphant.nginx/roles:../letsencrypt-nginx/roles
inventory:
links:
group_vars: ../../inventories/common/group_vars
host_vars: ../../inventories/common/host_vars
scenario:
name: api
test_sequence:
- destroy
- create
- converge
- verify
- destroy
verifier:
name: testinfra
options:
v: True
s: True
lint:
name: flake8

13
molecule/api/playbook.yml

@ -0,0 +1,13 @@
---
- import_playbook: ../firewall/firewall-playbook.yml
- import_playbook: ../letsencrypt/letsencrypt-playbook.yml
- import_playbook: ../misc/sshd-playbook.yml
- import_playbook: ../letsencrypt-nginx/letsencrypt-nginx-playbook.yml
- import_playbook: ../icinga/test-icinga-playbook.yml
- import_playbook: ../bind/bind-playbook.yml
- import_playbook: ../bind/bind-client-playbook.yml
- import_playbook: ../icinga/icinga-playbook.yml
- import_playbook: ../gitlab/gitlab-playbook.yml
- import_playbook: ../packages/packages-playbook.yml
- import_playbook: ../packages/enough-playbook.yml
- import_playbook: api-playbook.yml

2
molecule/api/roles/api/defaults/main.yml

@ -0,0 +1,2 @@
---
enough_admin_password: Jidak0ov_

3
molecule/api/roles/api/files/apt-preferences.d/default

@ -0,0 +1,3 @@
Package: *
Pin: release n=buster
Pin-Priority: -10

3
molecule/api/roles/api/files/apt-preferences.d/python3

@ -0,0 +1,3 @@
Package: python3
Pin: release n=buster
Pin-Priority: 942

151
molecule/api/roles/api/tasks/api.yml

@ -0,0 +1,151 @@
---
- name: apt-get install git, virtualenv, python-pip and python-setuptools
apt:
name: [ git, virtualenv, python-pip, python-setuptools ]
state: present
- name: pip install docker and docker-compose
pip:
name: [ docker, docker-compose ]
- name: apt-get install git
apt:
name:
- git
state: present
# - name: add buster repo and update cache
# apt_repository:
# repo: deb http://deb.debian.org/debian/ buster main
# state: present
# # workaround https://github.com/eliben/pycparser/issues/251
# - name: apt-get install python3-pycparser
# apt:
# name: python3-pycparser
# state: present
# - name: apt-get install python3-pip python3-dev python3-wheel python-setuptools libffi-dev libssl-dev gcc python-dev make
# apt:
# name: [python3-pip, python3-dev, python3-wheel, python-setuptools, libssl-dev, libffi-dev, gcc, python-dev, make]
# state: present
- name: get the api-host public key
command: cat /etc/ssh/ssh_host_rsa_key.pub
register: ssh_host_rsa_key_pub
changed_when: False
- name: Set up API ssh key to BIND authorized keys
authorized_key:
user: "{{ ansible_user }}"
state: present
key: "{{ ssh_host_rsa_key_pub.stdout }}"
delegate_to: bind-host
- name: mkdir /root/.enough
file:
path: "/root/.enough"
state: directory
- name: mkdir /root/.enough/default
file:
path: "/root/.enough/default"
state: directory
- name: cp /etc/ssh/ssh_host_rsa_key ~/.enough/default/id_rsa
copy:
remote_src: yes
src: /etc/ssh/ssh_host_rsa_key
dest: "/root/.enough/default/id_rsa"
owner: "{{ ansible_user }}"
mode: 0444
# - name: apt-get install rsync
# apt:
# name:
# - rsync
# state: present
# - name: copy the sources of enough
# synchronize:
# # root of the repository, relative to ../files
# src: ../../../../../
# rsync_opts:
# - "--delete-excluded"
# - "--exclude=.tox"
# - "--exclude=.#*"
# dest: /opt/enough
# - name: pip install enough
# pip:
# executable: pip3
# name: /opt/enough
# # name: git+https://lab.enough.community/singuliere/infrastructure@4f9c6e9c271d37401ee38ede3f42da7603541c6d#egg=enough
- name: get enough installation script
uri:
dest: /usr/local/bin/enough-build-docker-image.sh
url: https://packages.{{ domain }}/docker-enough/enough-build-docker-image.sh
- name: apt-get install curl
apt:
name:
- curl
state: present
- name: install enough
shell: |
bash -x /usr/local/bin/enough-build-docker-image.sh
docker run --rm enough install --script --no-version > /usr/local/bin/enough
chmod +x /usr/local/bin/enough
- name: enough manage migrate
shell: enough manage migrate
register: result
changed_when: '"Applying " in result.stdout'
- name: enough api server
shell: docker run --rm -v $HOME/.enough:/root/.enough -v /var/run/docker.sock:/var/run/docker.sock enough create service
# - name: enough manage createsuperuser
# shell: |
# enough manage shell
# args:
# stdin: |
# from django.contrib.auth import get_user_model
# User = get_user_model()
# if User.objects.filter(username='admin'):
# print('Already exists ' + str(User.objects.filter(username='admin')))
# else:
# User.objects.create_superuser('admin', 'admin@{{ domain }}', '{{ enough_admin_password }}')
# print('Created')
# register: result
# become: no
# changed_when: '"Created" in result.stdout'
# - name:
# docker_service:
# files: docker-compose-infrastructure.yml
# project_src: /srv/mattermost
# state: present
# become: False
# - name: enough manage runserver
# shell: |
# if netstat -tlpn | grep -q :8000 ; then
# echo 'Already running'
# else
# nohup enough manage runserver > /var/log/enough.log 2>&1 &
# echo 'Started'
# fi
# register: result
# changed_when: '"Started" in result.stdout'
#- name: setup enough
# shell: |
# enough manage migrate
# enough manage createsuperuser --email admin@example.com --username admin
# enough manage runserver
# echo '{"bind_host": "bind-host", "zone":"'$(hostname -d)'", "record":"foo.'$(hostname -d)'.", "ttl":"1800", "type":"A", "value": "1.2.3.4"}' | http -a 'admin:mypassword' POST http://127.0.0.1:8000/bind/

2
molecule/api/roles/api/tasks/main.yml

@ -0,0 +1,2 @@
---
- import_tasks: api.yml

35
molecule/api/tests/test_api.py

@ -0,0 +1,35 @@
import requests
import yaml
import dns.resolver
testinfra_hosts = ['api-host']
def get_domain():
vars_dir = '../../inventories/common/group_vars/all'
return yaml.load(open(vars_dir + '/domain.yml'))['domain']
# debug with
#
# molecule login -s api --host=api-host
# docker exec -ti tmp_enough-enough_1 journalctl -f --unit enough
#
def test_add_host(host):
domain = get_domain()
url = f"https://api.{domain}"
s = requests.Session()
s.verify = '../../certs'
data = {
"zone": domain,
"record": f"foo.{domain}.",
"ttl": "1800",
"type": "A",
"value": "1.2.3.4",
}
r = s.post(f'{url}/bind/', json=data, timeout=5)
r.raise_for_status()
resolver = dns.resolver.Resolver()
bind_ip = str(resolver.query(f'bind.{domain}.')[0])
resolver.nameservers = [bind_ip]
assert '1.2.3.4' == str(resolver.query(f'foo.{domain}.', 'a')[0])

55
molecule/packages/roles/enough-pip/tasks/enough-pip.yml

@ -16,29 +16,60 @@
path: /usr/share/nginx/html/enough
state: directory
owner: debian
when: enough_version is succeeded
when: enough_version.stdout != ''
- name: python setup.py sdist
shell: |
set -x
cd $(git rev-parse --show-toplevel)
version={{ enough_version.stdout }}
perl -pi -e "s/^version.*/version = $version/" setup.cfg
for i in 1 2 ; do
python setup.py sdist
amend=$(git log -1 --oneline | grep --quiet "version $version" && echo --amend)
git commit $amend -m "version $version" ChangeLog setup.cfg
git tag -a -f -m "version $version" $version
done
args:
creates: ../../dist/enough-{{ enough_version.stdout }}.tar.gz
git tag -a -f -m "version $version" $version
python setup.py sdist
git tag -d $version
git checkout ChangeLog
delegate_to: localhost
become: no
when: enough_version is succeeded
when: enough_version != ''
- name: cp enough-{{ enough_version.stdout }}.tar.gz
copy:
src: ../../../../../dist/enough-{{ enough_version.stdout }}.tar.gz
dest: /usr/share/nginx/html/enough/enough-{{ enough_version.stdout }}.tar.gz
become: no
when: enough_version is succeeded
when: enough_version != ''
- name: mkdir -p /usr/share/nginx/html/docker-enough
file:
path: /usr/share/nginx/html/docker-enough
state: directory
owner: debian
when: enough_version != ''
- name: cp enough/internal/data/enough.dockerfile
copy:
src: ../../../../../enough/internal/data/enough.dockerfile
dest: /usr/share/nginx/html/docker-enough/enough.dockerfile
become: no
when: enough_version != ''
- name: cp enough/common/data/base.dockerfile
copy:
src: ../../../../../enough/common/data/base.dockerfile
dest: /usr/share/nginx/html/docker-enough/base.dockerfile
become: no
when: enough_version != ''
- name: shell script to build image from this package host
copy:
content: |
d=$(mktemp -d)
cd $d
curl -q https://{{ packages_vhost_fqdn }}/docker-enough/base.dockerfile > Dockerfile
curl -q https://{{ packages_vhost_fqdn }}/docker-enough/enough.dockerfile >> Dockerfile
sed -i -e "s/replace this comment/$(date +%s)/" Dockerfile
docker build --build-arg PIP3_OPTS='--extra-index-url=https://{{ packages_vhost_fqdn }}/ --trusted-host={{ packages_vhost_fqdn }}' --build-arg ENOUGH_VERSION={{ enough_version.stdout }} -t enough .
cd -
rm -fr $d
dest: /usr/share/nginx/html/docker-enough/enough-build-docker-image.sh
become: no
when: enough_version != ''

2
molecule/preprod/molecule.yml

@ -4,6 +4,8 @@ driver:
lint:
name: yamllint
platforms:
- name: api-host
flavor: "s1-2"
- name: bind-host
flavor: "s1-2"
- name: postfix-host

39
tests/enough/api/test_api.py

@ -0,0 +1,39 @@
import pytest
from django.urls import reverse
from django.contrib.auth import get_user_model
from rest_framework.test import APITestCase
from rest_framework.test import APIClient
import mock
UserModel = get_user_model()
class APIUserAPITestCase(APITestCase):
@pytest.mark.django_db
def setUp(self):
self.user = UserModel.objects.create_user(
username='test', email='test@...', password='top_secret')
# token = Token.objects.create(user=self.user)
self.client = APIClient()
# self.client.credentials(HTTP_AUTHORIZATION='Token ' + token.key)
class TestReviewListUser(APIUserAPITestCase):
@pytest.mark.django_db
@mock.patch('enough.api.views.run_ansible')
def test_admin_can_patch_a_product(self, run_ansible):
url = reverse('bind')
run_ansible.return_value = '51.68.81.22 | FAILED! => {"changed": false}'
data = {
"bind_host": "51.68.81.22",
"zone": "gyztqojxhe3tinjrbi.test.enough.community",
"record": "foo.gyztqojxhe3tinjrbi.test.enough.community.",
"ttl": "1800",
"type": "A",
"value": "1.2.3.4",
}
response = self.client.post(url, data, format='json')
self.assertEqual(response.status_code, 201)

4
tests/enough/common/test_common_docker.py

@ -41,8 +41,8 @@ def test_replace_content():
docker_name = 'DOCKER_NAME'
with modified_environ('DOCKER_HOST'):
d = DockerFixture(docker_name)
before = 'Name = {{ this.get_image_name(None) }}'
after = 'Name = ' + d.get_image_name(None)
before = 'Name = {{ this.get_image_name_with_version(None) }}'
after = 'Name = ' + d.get_image_name_with_version(None)
assert d.replace_content(before) == after

4
tox.ini

@ -2,7 +2,9 @@
envlist = py3,flake8,docs
[testenv]
setenv = VIRTUAL_ENV={envdir}
setenv =
VIRTUAL_ENV={envdir}
DJANGO_SETTINGS_MODULE=enough.settings
usedevelop = True
install_command = pip install {opts} {packages}
deps =

Loading…
Cancel
Save