zuul/zuul
Code Issues Proposed changes

Add copy-keys and delete-keys zuul client commands

Browse Source 
These can be used when renaming a project.

Change-Id: I98cf304914449622f9db48651b83e0744b676498
This commit is contained in:
James E. Blair 2021-08-09 11:31:21 -07:00
parent 49d945b5bd
commit a0af6004de
5 changed files with 220 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
doc/source/reference/client.rst
View File

@ -224,3 +224,21 @@ import-keys
Example::
zuul import-keys /var/backup/zuul-keys.json
copy-keys
^^^^^^^^^
.. program-output:: zuul copy-keys --help
Example::
zuul copy-keys gerrit old_project gerrit new_project
delete-keys
^^^^^^^^^^^
.. program-output:: zuul delete-keys --help
Example::
zuul delete-keys gerrit old_project

6
releasenotes/notes/keystore-copy-67fbfb9108e75d04.yaml Normal file
View File

@ -0,0 +1,6 @@
---
features:
- |
The new commands ``zuul copy-keys`` and ``zuul delete-keys`` may
be useful when renaming projects in order to move the keys from
the old to the new project names without service interruption.

42
tests/unit/test_client.py
View File

@ -170,7 +170,7 @@ class TestWebTokenClient(BaseClientTestCase):
(token['exp'], now))
class TestKeyExportImport(ZuulTestCase):
class TestKeyOperations(ZuulTestCase):
tenant_config_file = 'config/single-tenant/main.yaml'
def test_export_import(self):
@ -211,3 +211,43 @@ class TestKeyExportImport(ZuulTestCase):
# Make sure the new data matches the original
new_data = self.getZKTree('/keystorage')
self.assertEqual(new_data, old_data)
def test_copy_delete(self):
config_file = os.path.join(self.test_root, 'zuul.conf')
with open(config_file, 'w') as f:
self.config.write(f)
p = subprocess.Popen(
[os.path.join(sys.prefix, 'bin/zuul'),
'-c', config_file,
'copy-keys',
'gerrit', 'org/project',
'gerrit', 'org/newproject',
],
stdout=subprocess.PIPE)
out, _ = p.communicate()
self.log.debug(out.decode('utf8'))
data = self.getZKTree('/keystorage')
self.assertEqual(
data['/keystorage/gerrit/org/org%2Fproject/secrets'],
data['/keystorage/gerrit/org/org%2Fnewproject/secrets'])
self.assertEqual(
data['/keystorage/gerrit/org/org%2Fproject/ssh'],
data['/keystorage/gerrit/org/org%2Fnewproject/ssh'])
p = subprocess.Popen(
[os.path.join(sys.prefix, 'bin/zuul'),
'-c', config_file,
'delete-keys',
'gerrit', 'org/project',
],
stdout=subprocess.PIPE)
out, _ = p.communicate()
self.log.debug(out.decode('utf8'))
data = self.getZKTree('/keystorage')
self.assertIsNone(
data.get('/keystorage/gerrit/org/org%2Fproject/secrets'))
self.assertIsNone(
data.get('/keystorage/gerrit/org/org%2Fproject/ssh'))

76
zuul/cmd/client.py
View File

@ -396,6 +396,41 @@ class Client(zuul.cmd.ZuulApp):
help='key export file path')
cmd_export_keys.set_defaults(func=self.export_keys)
cmd_copy_keys = subparsers.add_parser(
'copy-keys',
help='copy keys from one project to another',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''\
Copy secret keys from one project to another
When projects are renamed, this command may be used to
copy the secret keys from the current name to the new name
in order to avoid service interruption.'''))
cmd_copy_keys.set_defaults(command='copy-keys')
cmd_copy_keys.add_argument('src_connection', type=str,
help='original connection name')
cmd_copy_keys.add_argument('src_project', type=str,
help='original project name')
cmd_copy_keys.add_argument('dest_connection', type=str,
help='new connection name')
cmd_copy_keys.add_argument('dest_project', type=str,
help='new project name')
cmd_copy_keys.set_defaults(func=self.copy_keys)
cmd_delete_keys = subparsers.add_parser(
'delete-keys',
help='delete project keys',
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''\
Delete the ssh and secrets keys for a project
'''))
cmd_delete_keys.set_defaults(command='delete-keys')
cmd_delete_keys.add_argument('connection', type=str,
help='connection name')
cmd_delete_keys.add_argument('project', type=str,
help='project name')
cmd_delete_keys.set_defaults(func=self.delete_keys)
return parser
def parseArguments(self, args=None):
@ -817,6 +852,47 @@ class Client(zuul.cmd.ZuulApp):
import_data = json.load(f)
keystore.importKeys(import_data, self.args.force)
def copy_keys(self):
logging.basicConfig(level=logging.INFO)
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
args = self.args
# Load
ssh = keystore.loadProjectSSHKeys(args.src_connection,
args.src_project)
secrets = keystore.loadProjectsSecretsKeys(args.src_connection,
args.src_project)
# Save
keystore.saveProjectSSHKeys(args.dest_connection,
args.dest_project, ssh)
keystore.saveProjectsSecretsKeys(args.dest_connection,
args.dest_project, secrets)
self.log.info("Copied keys from %s %s to %s %s",
args.src_connection, args.src_project,
args.dest_connection, args.dest_project)
def delete_keys(self):
logging.basicConfig(level=logging.INFO)
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
args = self.args
keystore.deleteProjectSSHKeys(args.connection, args.project)
keystore.deleteProjectsSecretsKeys(args.connection, args.project)
self.log.info("Delete keys from %s %s",
args.connection, args.project)
def main():
Client().main()

107
zuul/lib/keystorage.py
View File

@ -16,6 +16,7 @@ import io
import json
import logging
import time
from contextlib import suppress
import cachetools
import kazoo
@ -83,11 +84,9 @@ class KeyStorage(ZooKeeperBase):
@cachetools.cached(cache={})
def getProjectSSHKeys(self, connection_name, project_name):
key_path = self.getSSHKeysPath(connection_name, project_name)
try:
key = self._getSSHKey(key_path)
except kazoo.exceptions.NoNodeError:
"""Return the public and private keys"""
key = self._getSSHKey(connection_name, project_name)
if key is None:
self.log.info("Generating a new SSH key for %s/%s",
connection_name, project_name)
key = paramiko.RSAKey.generate(bits=RSA_KEY_SIZE)
@ -95,11 +94,12 @@ class KeyStorage(ZooKeeperBase):
key_created = int(time.time())
try:
self._storeSSHKey(key_path, key, key_version, key_created)
self._storeSSHKey(connection_name, project_name, key,
key_version, key_created)
except kazoo.exceptions.NodeExistsError:
# Handle race condition between multiple schedulers
# creating the same SSH key.
key = self._getSSHKey(key_path)
key = self._getSSHKey(connection_name, project_name)
with io.StringIO() as o:
key.write_private_key(o)
@ -108,14 +108,39 @@ class KeyStorage(ZooKeeperBase):
return private_key, public_key
def _getSSHKey(self, key_path):
data, _ = self.kazoo_client.get(key_path)
keydata = json.loads(data)
def loadProjectSSHKeys(self, connection_name, project_name):
"""Return the complete internal data structure"""
key_path = self.getSSHKeysPath(connection_name, project_name)
try:
data, _ = self.kazoo_client.get(key_path)
return json.loads(data)
except kazoo.exceptions.NoNodeError:
return None
def saveProjectSSHKeys(self, connection_name, project_name, keydata):
"""Store the complete internal data structure"""
key_path = self.getSSHKeysPath(connection_name, project_name)
data = json.dumps(keydata).encode("utf-8")
self.kazoo_client.create(key_path, value=data, makepath=True)
def deleteProjectSSHKeys(self, connection_name, project_name):
"""Delete the complete internal data structure"""
key_path = self.getSSHKeysPath(connection_name, project_name)
with suppress(kazoo.exceptions.NoNodeError):
self.kazoo_client.delete(key_path)
def _getSSHKey(self, connection_name, project_name):
"""Load and return the public and private keys"""
keydata = self.loadProjectSSHKeys(connection_name, project_name)
if keydata is None:
return None
encrypted_key = keydata['keys'][0]["private_key"]
with io.StringIO(encrypted_key) as o:
return paramiko.RSAKey.from_private_key(o, self.password)
def _storeSSHKey(self, key_path, key, version, created):
def _storeSSHKey(self, connection_name, project_name, key,
version, created):
"""Create the internal data structure from the key and store it"""
# key is an rsa key object
with io.StringIO() as o:
key.write_private_key(o, self.password)
@ -129,8 +154,7 @@ class KeyStorage(ZooKeeperBase):
'schema': 1,
'keys': keys
}
data = json.dumps(keydata).encode("utf-8")
self.kazoo_client.create(key_path, value=data, makepath=True)
self.saveProjectSSHKeys(connection_name, project_name, keydata)
def getProjectSecretsKeysPath(self, connection_name, project_name):
key_project_name = strings.unique_project_name(project_name)
@ -139,12 +163,9 @@ class KeyStorage(ZooKeeperBase):
@cachetools.cached(cache={})
def getProjectSecretsKeys(self, connection_name, project_name):
key_path = self.getProjectSecretsKeysPath(
connection_name, project_name)
try:
pem_private_key = self._getSecretsKeys(key_path)
except kazoo.exceptions.NoNodeError:
"""Return the public and private keys"""
pem_private_key = self._getSecretsKey(connection_name, project_name)
if pem_private_key is None:
self.log.info("Generating a new secrets key for %s/%s",
connection_name, project_name)
private_key, public_key = encryption.generate_rsa_keypair()
@ -154,24 +175,55 @@ class KeyStorage(ZooKeeperBase):
key_created = int(time.time())
try:
self._storeSecretsKeys(key_path, pem_private_key,
key_version, key_created)
self._storeSecretsKey(connection_name, project_name,
pem_private_key, key_version,
key_created)
except kazoo.exceptions.NodeExistsError:
# Handle race condition between multiple schedulers
# creating the same secrets key.
pem_private_key = self._getSecretsKeys(key_path)
pem_private_key = self._getSecretsKey(
connection_name, project_name)
private_key, public_key = encryption.deserialize_rsa_keypair(
pem_private_key, self.password_bytes)
return private_key, public_key
def _getSecretsKeys(self, key_path):
data, _ = self.kazoo_client.get(key_path)
keydata = json.loads(data)
def loadProjectsSecretsKeys(self, connection_name, project_name):
"""Return the complete internal data structure"""
key_path = self.getProjectSecretsKeysPath(
connection_name, project_name)
try:
data, _ = self.kazoo_client.get(key_path)
return json.loads(data)
except kazoo.exceptions.NoNodeError:
return None
def saveProjectsSecretsKeys(self, connection_name, project_name, keydata):
"""Store the complete internal data structure"""
key_path = self.getProjectSecretsKeysPath(
connection_name, project_name)
data = json.dumps(keydata).encode("utf-8")
self.kazoo_client.create(key_path, value=data, makepath=True)
def deleteProjectsSecretsKeys(self, connection_name, project_name):
"""Delete the complete internal data structure"""
key_path = self.getProjectSecretsKeysPath(
connection_name, project_name)
with suppress(kazoo.exceptions.NoNodeError):
self.kazoo_client.delete(key_path)
def _getSecretsKey(self, connection_name, project_name):
"""Load and return the private key"""
keydata = self.loadProjectsSecretsKeys(
connection_name, project_name)
if keydata is None:
return None
return keydata['keys'][0]["private_key"].encode("utf-8")
def _storeSecretsKeys(self, key_path, key, version, created):
def _storeSecretsKey(self, connection_name, project_name, key,
version, created):
"""Create the internal data structure from the key and store it"""
# key is a pem-encoded (base64) private key stored in bytes
keys = [{
"version": version,
@ -182,5 +234,4 @@ class KeyStorage(ZooKeeperBase):
'schema': 1,
'keys': keys
}
data = json.dumps(keydata).encode("utf-8")
self.kazoo_client.create(key_path, value=data, makepath=True)
self.saveProjectsSecretsKeys(connection_name, project_name, keydata)