Add tool to export Rackspace DNS domains to bind format

This exports Rackspace DNS domains to bind format for backup and
migration purposes.

This installs a small tool to query and export all the domains we can
see via the Racksapce DNS API.

Because we don't want to publish the backups (it's the equivalent of a
zone xfer) it is run on, and logs output to, bridge.openstack.org from
cron once a day.

Change-Id: I50fd33f5f3d6440a8f20d6fec63507cb883f2d56

playbooks/roles/rax-dns-backup/README.rst

@@ -0,0 +1,4 @@
+Backup Rackspace managed DNS domain names
+
+Export a bind file for each of the domains used in the Rackspace
+managed DNS-as-a-service.

playbooks/roles/rax-dns-backup/files/rax-dns-backup

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+
+# Copyright 2020 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+#
+# Export domains for a given user/project
+#
+# Set auth values in the environment, or in a .ini file specified with
+# --config:
+#
+# RACKSPACE_USERNAME = used to login to web
+# RACKSPACE_PROJECT_ID = listed on account info
+# RACKSPACE_API_KEY = listed in the account details page
+#
+# By default exports all domains, filter the list with --domains=
+#
+
+import argparse
+import configparser
+import collections
+import datetime
+import glob
+import logging
+import os
+import requests
+import sys
+import time
+
+RACKSPACE_IDENTITY_ENDPOINT='https://identity.api.rackspacecloud.com/v2.0/tokens'
+RACKSPACE_DNS_ENDPOINT="https://dns.api.rackspacecloud.com/v1.0"
+
+RACKSPACE_PROJECT_ID=os.environ.get('RACKSPACE_PROJECT_ID', None)
+RACKSPACE_USERNAME=os.environ.get('RACKSPACE_USERNAME', None)
+RACKSPACE_API_KEY=os.environ.get('RACKSPACE_API_KEY', None)
+
+def get_auth_token(session):
+    # Get auth token
+    data = {'auth':
+            {
+                'RAX-KSKEY:apiKeyCredentials':
+                {
+                    'username': RACKSPACE_USERNAME,
+                    'apiKey': RACKSPACE_API_KEY
+                }
+            }
+    }
+    token_response = session.post(url=RACKSPACE_IDENTITY_ENDPOINT, json=data)
+    token = token_response.json()['access']['token']['id']
+
+    return token
+
+def get_domain_list(session, token):
+     # List all domains
+     domain_list_url = "%s/%s/domains" % (RACKSPACE_DNS_ENDPOINT,
+                                          RACKSPACE_PROJECT_ID)
+     headers = {
+         'Accept': 'application/json',
+         'X-Auth-Token': token,
+         'X-Project-Id': RACKSPACE_PROJECT_ID,
+         'Content-Type': 'application/json'
+     }
+     domain_list_response = session.get(url=domain_list_url, headers=headers)
+     return domain_list_response.json()['domains']
+
+def get_domain_id(session, token, domain):
+    # Find domain id
+    domain_url = "%s/%s/domains/search" % (RACKSPACE_DNS_ENDPOINT,
+                                           RACKSPACE_PROJECT_ID)
+    headers = {
+        'Accept': 'application/json',
+        'X-Auth-Token': token,
+        'X-Project-Id': RACKSPACE_PROJECT_ID,
+        'Content-Type': 'application/json'
+    }
+
+    query = {'name': domain}
+    domain_response = session.get(url=domain_url, params=query, headers=headers)
+    domains = domain_response.json()
+
+    for d in domains['domains']:
+        if d['name'] == domain:
+            return d
+
+    logging.error("Did not find domain: %s" % domain)
+    sys.exit(1)
+
+def do_bind_export(session, token, domain_id, outfile):
+    # export to file
+    headers = {
+        'Accept': 'application/json',
+        'X-Auth-Token': token,
+        'X-Project-Id': RACKSPACE_PROJECT_ID,
+        'Content-Type': 'application/json'
+    }
+
+    # Run export
+    export_url = '%s/%s/domains/%s/export' % (RACKSPACE_DNS_ENDPOINT,
+                                              RACKSPACE_PROJECT_ID,
+                                              domain_id)
+
+    # We get a callback URL; we should loop around and correctly
+    # detect the completed status and timeout and whatnot.  But we
+    # just sleep and that's enough.
+    export_response = session.get(url=export_url, headers=headers)
+    if export_response.status_code != 202:
+        logging.error("Didn't get export callback?")
+        sys.exit(1)
+    r = export_response.json()
+    callback_url = r['callbackUrl']
+    time.sleep(2)
+
+    query = {'showDetails': 'true'}
+    final_response = session.get(callback_url, params=query, headers=headers)
+
+    bind_output = final_response.json()['response']['contents']
+
+    output = []
+    for line in bind_output.split('\n'):
+        if line == '':
+            continue
+        fields = line.split(' ')
+        output.append(fields)
+
+    # find padding space for the first column
+    max_first = max([len(x[0]) for x in output])
+
+    # create a dict keyed by domain with each record
+    out_dict = collections.defaultdict(list)
+    for domain in output:
+        out_dict[domain[0]].append(domain[1:])
+
+    outstr = ''
+
+    # first output SOA then get rid of it
+    outstr += ("%-*s\t%s\n\n" %
+               (max_first+1, '@', '\t'.join(out_dict['@'][0]) ))
+    del(out_dict['@'])
+
+    # print out the rest of the entries, with individual records
+    # sorted and grouped
+    for domain in sorted(out_dict):
+        records = out_dict[domain]
+        # sort records by type
+        records.sort(key=lambda x: x[1])
+        for record in records:
+            outstr += ("%-*s\t%s\n" % (max_first+1, domain, '\t'.join(record) ))
+        outstr += '\n'
+
+    with open(outfile, 'w') as f:
+        f.write(outstr)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Dump Rackspace DNS domains')
+    parser.add_argument('--domains', dest='domains',
+                        help='Comma separated list of domains to export')
+    parser.add_argument('--output-dir', dest='output_dir',
+                        default='/var/lib/rax-dns-backup')
+    parser.add_argument('--config', dest='config',
+                        default='/etc/rax-dns-auth.conf')
+    parser.add_argument('--keep', dest='keep', type=int, default=30)
+    parser.add_argument('--debug', dest='debug', action='store_true')
+    args = parser.parse_args()
+
+    logging.basicConfig(level=logging.INFO)
+    if args.debug:
+        logging.getLogger().setLevel(logging.DEBUG)
+        requests_log = logging.getLogger("requests.packages.urllib3")
+        requests_log.setLevel(logging.DEBUG)
+        requests_log.propogate = True
+
+    logging.debug("Starting")
+
+    try:
+        logging.info("Reading config file %s" % args.config)
+        config = configparser.ConfigParser()
+        config.read(args.config)
+        global RACKSPACE_PROJECT_ID
+        global RACKSPACE_USERNAME
+        global RACKSPACE_API_KEY
+        RACKSPACE_PROJECT_ID = config['DEFAULT']['RACKSPACE_PROJECT_ID']
+        RACKSPACE_USERNAME = config['DEFAULT']['RACKSPACE_USERNAME']
+        RACKSPACE_API_KEY = config['DEFAULT']['RACKSPACE_API_KEY']
+    except:
+        logging.info("Skipping config read")
+
+    if (not RACKSPACE_PROJECT_ID) or \
+       (not RACKSPACE_USERNAME) or \
+       (not RACKSPACE_API_KEY):
+        logging.error("Must set auth variables!")
+        sys.exit(1)
+
+    if not os.path.isdir(args.output_dir):
+        logging.error("Output directory does not exist")
+        sys.exit(1)
+
+    session = requests.Session()
+    token = get_auth_token(session)
+
+    if args.domains:
+        to_dump = []
+        domains = args.domains.split(',')
+        for domain in domains:
+            logging.debug("Looking up domain: %s" % domain)
+            to_dump.append(get_domain_id(session, token, domain))
+    else:
+        to_dump = get_domain_list(session, token)
+
+    date_suffix = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.db")
+
+    for domain in to_dump:
+        outfile = os.path.join(
+            args.output_dir, "%s_%s" % (domain['name'], date_suffix))
+        logging.info("Dumping %s to %s" % (domain['name'], outfile))
+
+        do_bind_export(session, token, domain['id'], outfile)
+
+        # cleanup old runs
+        old_files = glob.glob(os.path.join(args.output_dir,
+                                           '%s_*.db' % domain['name']))
+        old_files.sort()
+        for f in old_files[:-args.keep]:
+            logging.info("Cleaning up old output: %s" % f)
+            os.unlink(f)
+
+if __name__ == "__main__":
+    main()
+

playbooks/roles/rax-dns-backup/tasks/main.yaml

@@ -0,0 +1,38 @@
+- name: Ensure configuration file
+  template:
+    src: rax-dns-auth.conf.j2
+    dest: /etc/rax-dns-auth.conf
+    owner: root
+    group: root
+    mode: 0600
+
+- name: Ensure output directory
+  file:
+    state: directory
+    path: /var/lib/rax-dns-backup
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Install backup tool
+  copy:
+    content: rax-dns-backup
+    dest: /usr/local/bin/rax-dns-backup
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Install cron job
+  cron:
+    name: 'Backup Rackspace DNS'
+    state: present
+    job: '/usr/local/bin/rax-dns-backup 2>&1 > /var/log/rax-dns-backup.log'
+    hour: '2'
+    minute: '0'
+    day: '*'
+
+- name: Install logrotate
+  include_role:
+    name: logrotate
+  vars:
+    logrotate_file_name: '/var/log/rax-dns-backup.log'

playbooks/roles/rax-dns-backup/templates/rax-dns-auth.conf.j2

@@ -0,0 +1,4 @@
+[DEFAULT]
+RACKSPACE_USERNAME={{ rackspace_dns_username }}
+RACKSPACE_PROJECT_ID={{ rackspace_dns_project_id }}
+RACKSPACE_API_KEY={{ rackspace_dns_api_key }}

playbooks/service-bridge.yaml

@@ -23,7 +23,12 @@
         name: configure-openstacksdk
       vars:
         openstacksdk_config_template: clouds/bridge_all_clouds.yaml.j2
+
     - name: Get rid of all-clouds.yaml
       file:
         state: absent
         path: '/etc/openstack/all-clouds.yaml'
+
+    - name: Install rackspace DNS backup tool
+      include_role:
+        name: rax-dns-backup

playbooks/zuul/templates/host_vars/bridge.openstack.org.yaml.j2

@@ -67,3 +67,7 @@ gitea_kube_key: Z2l0ZWFfazhzX2tleQ==
 ansible_cron_disable_job: true
 cloud_launcher_disable_job: true
 extra_users: []
+
+rackspace_dns_username: user
+rackspace_dns_project_id: 1234
+rackspace_dns_api_key: apikey

testinfra/test_bridge.py

@@ -91,3 +91,14 @@ def test_zuul_authorized_keys(host):
     assert len(keys) >= 2
     for key in keys:
         assert 'ssh-rsa' in key
+
+
+def test_rax_dns_backup(host):
+    config_file = host.file('/etc/rax-dns-auth.conf')
+    assert config_file.exists
+
+    tool_file = host.file('/usr/local/bin/rax-dns-backup')
+    assert tool_file.exists
+
+    output_dir = host.file('/var/lib/rax-dns-backup')
+    assert output_dir.exists