Add commands to export/import keys to/from ZK
This removes the filesystem-based keystore in favor of only using ZooKeeper. Zuul will no longer load missing keys from the filesystem, nor will it write out decrypted copies of all keys to the filesystem. This is more secure since it allows sites better control over when and where secret data are written to disk. To provide for system backups to aid in disaster recovery in the case that the ZK data store is lost, two new scheduler commands are added: * export-keys * import-keys These write the password-protected versions of the keys (in fact, a raw dump of the ZK data) to the filesystem, and read the same data back in. An administrator can invoke export-keys before performing a system backup, and run import-keys to restore the data. A minor doc change recommending the use of ``zuul-scheduler stop`` was added as well; this is left over from a previous version of this change but warrants updating. This also removes the test_keystore test file; key generation is tested in test_v3, and key usage is tested by tests which have encrypted secrets. Change-Id: I5e6ea37c94ab73ec6f850591871c4127118414ed
This commit is contained in:
James E. Blair
parent
a619c9d8a6
commit
49d945b5bd
|
|
@ -125,6 +125,15 @@ A minimal Zuul system may consist of a :ref:`scheduler` and
|
|||
should consider running multiple executors, each on a dedicated host,
|
||||
and running mergers on dedicated hosts as well.
|
||||
|
||||
Zuul stores private keys for each project it knows about in ZooKeeper.
|
||||
It is recommended that you periodically back up the private keys in
|
||||
case the ZooKeeper data store is lost or damaged. The :title:`Zuul
|
||||
Client` provides two sub-commands for use in this case:
|
||||
:title:`export-keys` and :title:`import-keys`. Each takes an argument to
|
||||
a filesystem path and will write the keys to, or read the keys from
|
||||
that path. The data in the exported files are still secured with the
|
||||
keystore passphrase, so be sure to retain it as well.
|
||||
|
||||
Common
|
||||
------
|
||||
|
||||
|
|
@ -405,8 +414,8 @@ The following sections of ``zuul.conf`` are used by the scheduler:
|
|||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the scheduler, run ``zuul-scheduler``. To stop it, kill the
|
||||
PID which was saved in the pidfile specified in the configuration.
|
||||
To start the scheduler, run ``zuul-scheduler``. To stop it, run
|
||||
``zuul-scheduler stop``.
|
||||
|
||||
Reconfiguration
|
||||
~~~~~~~~~~~~~~~
|
||||
|
|
|
|||
|
|
@ -206,3 +206,21 @@ when querying a protected endpoint on Zuul's REST API.
|
|||
Example::
|
||||
|
||||
bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vbWFuYWdlc2Yuc2ZyZG90ZXN0aW5zdGFuY2Uub3JnIiwienV1bC50ZW5hbnRzIjp7ImxvY2FsIjoiKiJ9LCJleHAiOjE1Mzc0MTcxOTguMzc3NTQ0fQ.DLbKx1J84wV4Vm7sv3zw9Bw9-WuIka7WkPQxGDAHz7s
|
||||
|
||||
export-keys
|
||||
^^^^^^^^^^^
|
||||
|
||||
.. program-output:: zuul export-keys --help
|
||||
|
||||
Example::
|
||||
|
||||
zuul export-keys /var/backup/zuul-keys.json
|
||||
|
||||
import-keys
|
||||
^^^^^^^^^^^
|
||||
|
||||
.. program-output:: zuul import-keys --help
|
||||
|
||||
Example::
|
||||
|
||||
zuul import-keys /var/backup/zuul-keys.json
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
upgrade:
|
||||
- |
|
||||
Zuul no longer reads or writes project private key files from the
|
||||
scheduler's filesystem. In order to load existing keys into
|
||||
ZooKeeper, run version 4.6.0 of the scheduler at least once, if
|
||||
you haven't already.
|
||||
|
||||
A new command ``zuul export-keys`` has been added to export the
|
||||
encrypted keys from ZooKeeper onto the filesystem for backup.
|
||||
Likewise, ``zuul import-keys`` will load a previously-exported
|
||||
backup into ZooKeeper. It is recommended that you use these
|
||||
commands in system backup scripts.
|
||||
|
|
@ -111,6 +111,7 @@ import zuul.executor.client
|
|||
import zuul.lib.ansible
|
||||
import zuul.lib.connections
|
||||
import zuul.lib.auth
|
||||
import zuul.lib.keystorage
|
||||
import zuul.merger.client
|
||||
import zuul.merger.merger
|
||||
import zuul.merger.server
|
||||
|
|
@ -4236,13 +4237,19 @@ class BaseTestCase(testtools.TestCase):
|
|||
sessions = None
|
||||
return ret
|
||||
|
||||
def getZKTree(self, root):
|
||||
items = []
|
||||
for x in self.zk_client.client.get_children(root):
|
||||
path = '/'.join([root, x])
|
||||
items.append(path)
|
||||
items.extend(self.getZKTree(path))
|
||||
return items
|
||||
def getZKTree(self, path, ret=None):
|
||||
"""Return the contents of a ZK tree as a dictionary"""
|
||||
if ret is None:
|
||||
ret = {}
|
||||
for key in self.zk_client.client.get_children(path):
|
||||
subpath = os.path.join(path, key)
|
||||
ret[subpath] = self.zk_client.client.get(
|
||||
os.path.join(path, key))[0]
|
||||
self.getZKTree(subpath, ret)
|
||||
return ret
|
||||
|
||||
def getZKPaths(self, path):
|
||||
return list(self.getZKTree(path).keys())
|
||||
|
||||
|
||||
class SymLink(object):
|
||||
|
|
@ -4533,8 +4540,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
'scheduler', 'command_socket',
|
||||
os.path.join(self.test_root, 'scheduler.socket'))
|
||||
if not self.config.has_option("keystore", "password"):
|
||||
self.config.set("keystore", "password",
|
||||
uuid.uuid4().hex)
|
||||
self.config.set("keystore", "password", 'keystorepassword')
|
||||
self.config.set('merger', 'git_dir', self.merger_src_root)
|
||||
self.config.set('executor', 'git_dir', self.executor_src_root)
|
||||
self.config.set('executor', 'private_key_file', self.private_key_file)
|
||||
|
|
@ -4603,6 +4609,9 @@ class ZuulTestCase(BaseTestCase):
|
|||
self.connection_event_queues = DefaultKeyDict(
|
||||
lambda cn: ConnectionEventQueue(self.zk_client, cn)
|
||||
)
|
||||
# requires zk client
|
||||
self.setupAllProjectKeys(self.config)
|
||||
|
||||
self.poller_events = {}
|
||||
self._configureSmtp()
|
||||
self._configureMqtt()
|
||||
|
|
@ -4754,7 +4763,6 @@ class ZuulTestCase(BaseTestCase):
|
|||
os.path.join(git_path, reponame))
|
||||
# Make test_root persist after ansible run for .flag test
|
||||
config.set('executor', 'trusted_rw_paths', self.test_root)
|
||||
self.setupAllProjectKeys(config)
|
||||
|
||||
return config
|
||||
|
||||
|
|
@ -4840,38 +4848,25 @@ class ZuulTestCase(BaseTestCase):
|
|||
def setupProjectKeys(self, source, project):
|
||||
# Make sure we set up an RSA key for the project so that we
|
||||
# don't spend time generating one:
|
||||
|
||||
if isinstance(project, dict):
|
||||
project = list(project.keys())[0]
|
||||
key_root = os.path.join(self.state_root, 'keys')
|
||||
if not os.path.isdir(key_root):
|
||||
os.mkdir(key_root, 0o700)
|
||||
fn = os.path.join(key_root, '.version')
|
||||
with open(fn, 'w') as f:
|
||||
f.write('1')
|
||||
# secrets key
|
||||
private_key_file = os.path.join(
|
||||
key_root, 'secrets', 'project', source, project, '0.pem')
|
||||
private_key_dir = os.path.dirname(private_key_file)
|
||||
self.log.debug("Installing test secrets keys for project %s at %s" % (
|
||||
project, private_key_file))
|
||||
if not os.path.isdir(private_key_dir):
|
||||
os.makedirs(private_key_dir)
|
||||
with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
|
||||
with open(private_key_file, 'w') as o:
|
||||
o.write(i.read())
|
||||
|
||||
password = self.config.get("keystore", "password")
|
||||
keystore = zuul.lib.keystorage.KeyStorage(
|
||||
self.zk_client, password=password)
|
||||
import_keys = {}
|
||||
import_data = {'keys': import_keys}
|
||||
|
||||
path = keystore.getProjectSecretsKeysPath(source, project)
|
||||
with open(os.path.join(FIXTURE_DIR, 'secrets.json'), 'rb') as i:
|
||||
import_keys[path] = json.load(i)
|
||||
|
||||
# ssh key
|
||||
private_key_file = os.path.join(
|
||||
key_root, 'ssh', 'project', source, project, '0.pem')
|
||||
private_key_dir = os.path.dirname(private_key_file)
|
||||
self.log.debug("Installing test ssh keys for project %s at %s" % (
|
||||
project, private_key_file))
|
||||
if not os.path.isdir(private_key_dir):
|
||||
os.makedirs(private_key_dir)
|
||||
with open(os.path.join(FIXTURE_DIR, 'ssh.pem')) as i:
|
||||
with open(private_key_file, 'w') as o:
|
||||
o.write(i.read())
|
||||
path = keystore.getSSHKeysPath(source, project)
|
||||
with open(os.path.join(FIXTURE_DIR, 'ssh.json'), 'rb') as i:
|
||||
import_keys[path] = json.load(i)
|
||||
|
||||
keystore.importKeys(import_data, False)
|
||||
|
||||
def copyDirToRepo(self, project, source_path):
|
||||
self.init_repo(project)
|
||||
|
|
@ -5424,12 +5419,12 @@ class ZuulTestCase(BaseTestCase):
|
|||
def assertCleanZooKeeper(self):
|
||||
# Make sure there are no extraneous ZK nodes
|
||||
client = self.merger_api
|
||||
self.assertEqual(self.getZKTree(client.REQUEST_ROOT), [])
|
||||
self.assertEqual(self.getZKTree(client.PARAM_ROOT), [])
|
||||
self.assertEqual(self.getZKTree(client.RESULT_ROOT), [])
|
||||
self.assertEqual(self.getZKTree(client.RESULT_DATA_ROOT), [])
|
||||
self.assertEqual(self.getZKTree(client.WAITER_ROOT), [])
|
||||
self.assertEqual(self.getZKTree(client.LOCK_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.REQUEST_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.PARAM_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.RESULT_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.RESULT_DATA_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.WAITER_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.LOCK_ROOT), [])
|
||||
|
||||
def assertReportedStat(self, key, value=None, kind=None, timeout=5):
|
||||
"""Check statsd output
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
{"schema": 1, "keys": [{"version": 0, "created": 1626909706, "private_key": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,8448A7DF16A5CF040EA107EB6B0A5235\n\ncLcAbSpqh3pnH5Q8ATw47yPmjAoPbDwq52mxDx53PmD0DfYw6u197jhNxRZmedu4\nvRk226Gr4ppsvj62LKK6Na95nLl+S9pYHKv6ZI4gVx8sliQDthgpWbYt/gYE9Wk2\n/wcZ58iQJTFzjM0LrhOgx4HXuuohKhlIjDj/sdo5cJiva4wNyxJlgEcjbMsLBlDG\no2Li7Wn9wC/5wKrG184yHbJJhbTV+FtGJeMcJLfy0l6Kod6jnYi1p5O+ZlMCBnqC\nF1VjqjeF95uvO075V33DFsdFdDWpSWHOQvLmzX2V0izNqW+06jurkS948hBYnJRK\ntL6waUzjLOoU8fEPfEoxVOCJssEsqgSgezE0YL4w7l/VSyvO+G/AS0uYcpCMT4dV\n9rZJ0WGwKgvfpVysR0hREBQ45zxuxo3Qh74GJKM10c2j1FPBk/M2YUhje54yHJ4a\nvWrmIXQLCQ9PtiMdWKaijIhPce0l8pRSomYy4kwZFzhLlzKCd1BzLpT4TwIm1P0H\nToyZu4zIT9fGdfX+WpZRbUnvIFVSmTpMJ47Yp21ScVJGN5ezgNfe8Eih2rgaAB0C\nZ1bXtYJxPkYmWPZn8llk80d5uLKDHJDMpL8kge+Km1MyTPayZRVknG4z160ukTTv\nv/nLfMmzeD9BuGFPSAMjNEntc0ld/ntVntvysLP3NLJguu4T0QTvbMJjRPfosKIv\n58PLCb9eQHAArSdkqV1ybMqpFATGdnbTg5uHodpGAoDZp7l8cfgO43fUTleIdRri\nVeNdnH4iSedcZ/SZBy13xmC7MUEwotToAHxHPiKvoonvq1gv4YWZyMio50ySKR5u\neJmEQZ2+Veiqk3a6ldxhjCuAx6E4iwOZWsASA6GhfhZNOv/oUqAjguLKS6bzCuBn\nnfp9MHHdsf+FfJJ1/OXYmvnVz9UB3bQna/+e/KJd4Ntj1D8IJj8OMsT3Kjsz3yJ9\n3CVJYnfAH1sS/7qMyv1hDTcDOy7EwbHc3LznKo/0PUuO9Rvm+eoyzM16XAvOW4Zu\nKALuC98FUtUKO1TpuZU59w5NkwCoWMnulVgYPjRNOymPUgFpiopcbCQmRJRDJ0hp\n5syncDp5XGiq51TXZ9nXQRS2c0TNIHlR+hT3NvJ+H/hO7ZjmiA7uVsdRubhJrQsv\nqZMACSkhY4etnDWVXW7G90eBBHeT6hobVwSHobaxkYkMmPD2Nrmsoh9a6y3Kpd8S\n0BVdQA6vNig85B3bP640otL9jIN52tLAoA//CvFkiIeZunBIWqfEymlhe06j0LGP\ntDtvF+3vqoyIGod0kCj47czEe7f/vhDyzAjHKU13vqrUdVPOdiAUUPMvpppdt0Iy\nODMFRIZqEOZXFTu08YiB22O32wFWy8rPvPSUcva+1dV6VKW0m51BkBgDeczqearn\nNjMyh1TA2KayeZXj+CaACUfgrgtgwZ2XwbZLHudRbCTOpGtrVtCQmdgE9qHFaFED\nW22MHkQ3dR8NPx9XwQIGRMYS6SavrpPsgI5J4EnQ3F8nxcBK7vWWcL5cBCixPFVz\n30bDcsx8HUoQUzjJEqMwZ/vnuElgBfYpjYlAP3hWZp/BGQr3dlUisG1c6bfhoefq\nNNl3Pubt1tj72I6V43OqUAF1zxsMiXLmRR0Eqcyd2VA2SZ+n3gD5aEmJ2pfGNESW\npgRSNA/BuB2ToRTYWo1yW//FNSmuNPwYPqB4rcG4F7m3/W3cV6olXCL6pFHKKjva\nnNU+Gn2TCoVTbmSRab9EXgO5TbUybFU/RV4OAgOhkIFe32Tg7ksp/LYBuytQ8kAG\n19ve5b5xC7LhEbEJEl435eU8coAavPW9+BMGuQUlDkg7Kq3N29HeDZ+OpAy4FJDu\nwkNb68zno1f7F71ZgkHH/AKRwvq5ut+TFA91Vk8e+w+N6ftszjJFs7917TENLKW/\n0+eAlYFY+bHmqxjJbEnfCMNMBMz5H7lHsOZmz+TGL8+DTPzj0t+PoSO15xi4Ga7F\n1X3UQQxQkA42nqGxhVMY7SE/wTRwT02ZUKPEus1IsiA73uZNJst3q5ddwzt8DkfU\n/Ov2m6M/PDnwn26tZQnNr1GK6jHMtnTa6xQDeXHUMymxbRc6jE71cPuHVEZgsfJo\n54f1vrInRDDPb2gjydRtxDIk3Fd5apBLZymadSqqnm6G4LLzCVQ8TSA6Rrya3J3V\nQ1gD9wEwhRmwtheNAV/qdBk3KCYAJAfnQAANuT87oyeJAEiPRKSSIACd9zkS/tq1\nptjeFb4y+Gs2x5qPzGUHICvBteuS6h/kdNMGBzcwdntXPV1hM4lafnCj9VCMCF1k\nwPz/M740VY6zdxo8d2LYuxKpUYcTzkZbuJorKGVgW1I4EkfAXF2BToAQ8kKsKVsK\nuDOo+s8ivAswlAX4gCtvVnESqTALwmFbJsq8OeleT4tSiNeIwhZM0sh3VdztD0Jw\nAJOULbcN08wYilFb+bgvzDlEIauiEv/8agmyTf3j+7q9FpPPzGAMrQbLZ1yhx8IN\n9u+ks9Xp3vtctJXMKJVyUVUYFGEEK8hbHNazLmpo2N7pV90uWlKfPGo9iF9osd1V\nwERF/INtO9gyPtsQNMylbvcVouAEx+A/q3+UtUIQyuc9t6+RWlxpVgFDTjylJsuI\nP3NYxt7kM9OP8TPTHoooQd28EOIVgzsXS8OmtKgTxScaU09/6EaAFPjWwVZJaA0E\nqBXlS7SkucEw7YU5Qx65y0B+r/keIdu7Cvc3XXyeISKZTzo94VIJvCC/cTIMZY/K\n9vnPJxnwmj0EnHB8Wb2j8FZqEIpDwh96mTWoroX4PGZdnKaaxL1vQISlss4KQ3y4\nJGVRyauK5rC9SwBWBCiZCPEHZ5q82m8IT3w41/umY4S01a33D1pgTa1rZEj38eVU\n6n9dP67kjP1C7m3YIDxNSX9+yv1sx8PBZ6ixomj/KAeXNbWzveDwPJM4H09qyvKU\nAhVPwIdUuJMbXsjumjxXXSOw2+A8KwAyPQ8vry3lshSP7QvtDOkCY33G3iOJJoQ1\nyv8Y8Hw6GPVrzGnjkYKRNfwlWirfad6e9sHGxv2VrH4bZzulXwnobZ5FpXJUv8s7\ndICGLSBrxozAoZiMLVyp3MMeo/iudcIIPb423R8VNiAiMxLpMezqbYEnZezzRyJo\n-----END RSA PRIVATE KEY-----\n"}]}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"schema": 1, "keys": [{"version": 0, "created": 1626909706, "private_key": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,7FF563CF60708DF4F2999DF100EC742D\n\nqecR3BS3nKFoKuexputN2ubp1nfHwFWw6VvD4wb3uBXxgzUeaSP9qK7qEUJU6bU9\nPvTJTPLrnNTD0zwsVyw2uJEv3PTFZ7tCVEocUNP9LoGrta8xXZ3QGDzo3rJ6GPCK\nR4UsNZa4KStnqAtcxw5nhhGXPZT0FDBsT9dgSi6JeYZZIifojWZG+jaoSXEaRe+O\nF34ys73gsmWo9tOvW8mN+YuFTKc8TlCviIa+NnqDHotF7tNp8SxIPJqBE4lygiEo\nzuHKJOAlohvLqVQsKk1383w5Xj8nKO33EhsAidqy4VBwnu/e4pbMIgdBkCFqHiVb\nTIhb452LMpaZr/fVge8qOXr/oULLXkGppVVxza0M+9fnYIwjEDryrFy0JTk88JIg\n3tw9CSn6DEN8yDXtkZZnOvcbJKrlOZll3e1A7jJpVAALM/NdVbGqVq2q9bWCf+Ql\nCHY0g9DAd//0/wgl2IRQ1jfOPMIJATsgrj7h6g4vj/Fsv9vKO9LPO9wILAlHHEc9\nYd0rBDT7nLTgdRNghf+auRBvi8fZxfbGXuv86Ei1A9iS/bf6+SlVt8V8tHd9/mnx\nq7PHlwUXg3IPONn0U+o6/uIJKW+t55PdmhXUhPc1TbvVH3QbiywzzR+q+lw5QhgO\nqJqKiJhEsfSuAk5B2/xBQxQpuFGs+8MH/AaapUu8rD+skZphWnxZZLmVYHCG111o\ny36RmhSBXkWy8wwpM9VXyDAJ9WhjE0VpFX5/ZsST+UwZIeGmL6NU4NQ9jYS2KAGj\nfDRlU3O4iA92kCYyj/tNeetkj+pDO6otYdUSwbblE8jyCqulV2IYCabDe+cfauym\nTMaBdFNRw9zB0poqpNqbHflGihuoN7Gb37ST7ILNsqPLqIP52oRS/JHHeVuaiQsJ\naUM0L+rOOxRImkEukp0IS7I8X5m+KJ8tRe5s1i3ojBykGpCtodxLGcxEebM9ncC+\n3U2m4nsyBMeiqVd2DJgwi7Qep4YTUXwzSxzGxLNTXQfokN598pi9RTu0AcTf0/Rs\n4a7sOavbuRJMEAzvoDDHOAfm39qVP4HwJ0142Cg9Y0P6VUgJSGXSagsRwPIFSOpJ\nnanC8sANMG4zSK4Aujyf4S/roK4uK3p5nkmF+sn/bD7XD4ACYBwBos+cQxOBRkiZ\nOxEevbP2NWlkMFU8P8HhVJQyv8YvV+K2tWYE3TS/v/DUhE22sObepzwkE/69sZWb\nBTEMsZEIaC1nH0JDLJ7PQqRgcRWu8gzsTUQZtd9VPypenE1XmJp/9jPqAAa5Fa3v\nZPnUPDnykUBaiLKINb4daMcf4MMJ4F1cHB8hbGKkDqNoUIa0U8MT/t8aeSWquSyk\nCKHbIqDHJESn/04YPFrSEx/36bP77DiiA7SKI/YjqSF7h+QXGwAiO7P2FBD05YST\nkQtDwr2vPH2w2PK2aqfacedZREYROWh4XzCYuLjFd7dNQ8QhKkxtcX2RxR6EJHAf\nYgaCyz2Il9jkR+7e3uVNDwv5aUIPqa/+PiNQzhNR9nrP513cxuWbdZlIemft11MO\nwY53ubT/oiRK1K+5/ZNZUt5REVpLr1Kls8nrOACL2RlfFrtizmRslIOCGIDIIg9M\n-----END RSA PRIVATE KEY-----\n"}]}
|
||||
|
|
@ -21,8 +21,10 @@ import time
|
|||
import configparser
|
||||
import fixtures
|
||||
import jwt
|
||||
import testtools
|
||||
from kazoo.exceptions import NoNodeError
|
||||
|
||||
from tests.base import BaseTestCase
|
||||
from tests.base import BaseTestCase, ZuulTestCase
|
||||
from tests.base import FIXTURE_DIR
|
||||
|
||||
|
||||
|
|
@ -166,3 +168,46 @@ class TestWebTokenClient(BaseClientTestCase):
|
|||
# allow one minute for the process to run
|
||||
self.assertTrue(580 <= int(token['exp']) - now < 660,
|
||||
(token['exp'], now))
|
||||
|
||||
|
||||
class TestKeyExportImport(ZuulTestCase):
|
||||
tenant_config_file = 'config/single-tenant/main.yaml'
|
||||
|
||||
def test_export_import(self):
|
||||
# Test a round trip export/import of keys
|
||||
export_root = os.path.join(self.test_root, 'export')
|
||||
config_file = os.path.join(self.test_root, 'zuul.conf')
|
||||
with open(config_file, 'w') as f:
|
||||
self.config.write(f)
|
||||
|
||||
# Save a copy of the keys in ZK
|
||||
old_data = self.getZKTree('/keystorage')
|
||||
|
||||
# Export keys
|
||||
p = subprocess.Popen(
|
||||
[os.path.join(sys.prefix, 'bin/zuul'),
|
||||
'-c', config_file,
|
||||
'export-keys', export_root],
|
||||
stdout=subprocess.PIPE)
|
||||
out, _ = p.communicate()
|
||||
self.log.debug(out.decode('utf8'))
|
||||
|
||||
# Delete keys from ZK
|
||||
self.zk_client.client.delete('/keystorage', recursive=True)
|
||||
|
||||
# Make sure it's really gone
|
||||
with testtools.ExpectedException(NoNodeError):
|
||||
self.getZKTree('/keystorage')
|
||||
|
||||
# Import keys
|
||||
p = subprocess.Popen(
|
||||
[os.path.join(sys.prefix, 'bin/zuul'),
|
||||
'-c', config_file,
|
||||
'import-keys', export_root],
|
||||
stdout=subprocess.PIPE)
|
||||
out, _ = p.communicate()
|
||||
self.log.debug(out.decode('utf8'))
|
||||
|
||||
# Make sure the new data matches the original
|
||||
new_data = self.getZKTree('/keystorage')
|
||||
self.assertEqual(new_data, old_data)
|
||||
|
|
|
|||
|
|
@ -1,182 +0,0 @@
|
|||
# Copyright 2018 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import fixtures
|
||||
from unittest.mock import patch
|
||||
|
||||
from zuul.lib import encryption
|
||||
from zuul.lib import keystorage
|
||||
from zuul.zk import ZooKeeperClient
|
||||
|
||||
from tests.base import BaseTestCase
|
||||
|
||||
|
||||
class TestFileKeyStorage(BaseTestCase):
|
||||
|
||||
def _setup_keys(self, root, connection_name, project_name):
|
||||
cn = os.path.join(root, connection_name)
|
||||
if '/' in project_name:
|
||||
pn = os.path.join(cn, os.path.dirname(project_name))
|
||||
os.makedirs(pn)
|
||||
fn = os.path.join(cn, project_name + '.pem')
|
||||
with open(fn, 'w'):
|
||||
pass
|
||||
|
||||
def assertFile(self, root, path, contents=None):
|
||||
fn = os.path.join(root, path)
|
||||
self.assertTrue(os.path.exists(fn))
|
||||
if contents:
|
||||
with open(fn) as f:
|
||||
self.assertEqual(contents, f.read())
|
||||
|
||||
def assertPaths(self, root, paths):
|
||||
seen = set()
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
for d in dirnames:
|
||||
seen.add(os.path.join(dirpath[len(root) + 1:], d))
|
||||
for f in filenames:
|
||||
seen.add(os.path.join(dirpath[len(root) + 1:], f))
|
||||
self.assertEqual(set(paths), seen)
|
||||
|
||||
def test_key_storage(self):
|
||||
root = self.useFixture(fixtures.TempDir()).path
|
||||
self._setup_keys(root, 'gerrit', 'org/example')
|
||||
keystorage.FileKeyStorage(root)
|
||||
self.assertFile(root, '.version', '1')
|
||||
self.assertPaths(root, [
|
||||
'.version',
|
||||
'secrets',
|
||||
'secrets/project',
|
||||
'secrets/project/gerrit',
|
||||
'secrets/project/gerrit/org',
|
||||
'secrets/project/gerrit/org/example',
|
||||
'secrets/project/gerrit/org/example/0.pem',
|
||||
'ssh',
|
||||
'ssh/project',
|
||||
'ssh/tenant',
|
||||
])
|
||||
# It shouldn't need to upgrade this time
|
||||
keystorage.FileKeyStorage(root)
|
||||
|
||||
|
||||
class TestZooKeeperKeyStorage(BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
self.setupZK()
|
||||
self.zk_client = ZooKeeperClient(
|
||||
self.zk_chroot_fixture.zk_hosts,
|
||||
tls_cert=self.zk_chroot_fixture.zookeeper_cert,
|
||||
tls_key=self.zk_chroot_fixture.zookeeper_key,
|
||||
tls_ca=self.zk_chroot_fixture.zookeeper_ca)
|
||||
self.addCleanup(self.zk_client.disconnect)
|
||||
self.zk_client.connect()
|
||||
|
||||
def test_backup(self):
|
||||
root = self.useFixture(fixtures.TempDir()).path
|
||||
backup = keystorage.FileKeyStorage(root)
|
||||
key_store = keystorage.ZooKeeperKeyStorage(
|
||||
self.zk_client, password="DEADBEEF", backup=backup)
|
||||
|
||||
# Create keys in the backup keystore
|
||||
backup_secrets_pk = encryption.serialize_rsa_private_key(
|
||||
backup.getProjectSecretsKeys("github", "org/project")[0])
|
||||
backup_ssh_keys = backup.getProjectSSHKeys("github", "org/project")
|
||||
|
||||
self.assertEqual(
|
||||
encryption.serialize_rsa_private_key(
|
||||
key_store.getProjectSecretsKeys("github", "org/project")[0]
|
||||
), backup_secrets_pk)
|
||||
self.assertEqual(
|
||||
key_store.getProjectSSHKeys("github", "org/project"),
|
||||
backup_ssh_keys)
|
||||
|
||||
# Keys should initially not be in the backup keystore
|
||||
self.assertFalse(
|
||||
backup.hasProjectSecretsKeys("github", "org/project1"))
|
||||
self.assertFalse(
|
||||
backup.hasProjectSSHKeys("github", "org/project1"))
|
||||
|
||||
self.assertIsNotNone(
|
||||
key_store.getProjectSecretsKeys("github", "org/project1"))
|
||||
self.assertIsNotNone(
|
||||
key_store.getProjectSSHKeys("github", "org/project1"))
|
||||
|
||||
# Keys should now also exist in the backup keystore
|
||||
self.assertTrue(
|
||||
backup.hasProjectSecretsKeys("github", "org/project1"))
|
||||
self.assertTrue(
|
||||
backup.hasProjectSSHKeys("github", "org/project1"))
|
||||
|
||||
def test_key_store_upgrade(self):
|
||||
# Test that moving an unencrypted key on the file system to ZK
|
||||
# (encrypted) works as expected and the backup keys stay the same.
|
||||
root = self.useFixture(fixtures.TempDir()).path
|
||||
backup = keystorage.FileKeyStorage(root)
|
||||
key_store = keystorage.ZooKeeperKeyStorage(
|
||||
self.zk_client, password="DECAFBAD", backup=backup)
|
||||
|
||||
# Create reference keys in the backup key store
|
||||
ref_secrets_pk = encryption.serialize_rsa_private_key(
|
||||
backup.getProjectSecretsKeys("github", "org/project")[0])
|
||||
ref_ssh_keys = backup.getProjectSSHKeys("github", "org/project")
|
||||
|
||||
# Make sure we get the backup keys via the primary key store.
|
||||
self.assertEqual(
|
||||
encryption.serialize_rsa_private_key(
|
||||
key_store.getProjectSecretsKeys("github", "org/project")[0]
|
||||
), ref_secrets_pk)
|
||||
self.assertEqual(key_store.getProjectSSHKeys("github", "org/project"),
|
||||
ref_ssh_keys)
|
||||
|
||||
# Make sure we can still read the updates keys from the backup
|
||||
# key store.
|
||||
self.assertEqual(
|
||||
encryption.serialize_rsa_private_key(
|
||||
backup.getProjectSecretsKeys("github", "org/project")[0]
|
||||
), ref_secrets_pk)
|
||||
self.assertEqual(backup.getProjectSSHKeys("github", "org/project"),
|
||||
ref_ssh_keys)
|
||||
|
||||
# Make sure that the backup key store is not used after keys have been
|
||||
# written to the primary key store.
|
||||
exc = AssertionError("Keys should not be loaded from backup store")
|
||||
# with patch.object(backup, "getProjectSecretsKeys", side_effect=exc):
|
||||
with patch.object(backup, "getProjectSecretsKeys", side_effect=exc):
|
||||
self.assertEqual(
|
||||
encryption.serialize_rsa_private_key(
|
||||
key_store.getProjectSecretsKeys("github", "org/project")[0]
|
||||
), ref_secrets_pk)
|
||||
|
||||
exc = AssertionError("SSH keys should not be loaded from backup store")
|
||||
with patch.object(backup, "getProjectSSHKeys", side_effect=exc):
|
||||
self.assertEqual(
|
||||
key_store.getProjectSSHKeys("github", "org/project"),
|
||||
ref_ssh_keys)
|
||||
|
||||
def test_without_backup(self):
|
||||
key_store = keystorage.ZooKeeperKeyStorage(
|
||||
self.zk_client, password="DECAFBAD")
|
||||
secrets_pk = encryption.serialize_rsa_private_key(
|
||||
key_store.getProjectSecretsKeys("github", "org/project")[0])
|
||||
ssh_keys = key_store.getProjectSSHKeys("github", "org/project")
|
||||
|
||||
self.assertEqual(
|
||||
encryption.serialize_rsa_private_key(
|
||||
key_store.getProjectSecretsKeys("github", "org/project")[0]
|
||||
), secrets_pk)
|
||||
self.assertEqual(key_store.getProjectSSHKeys("github", "org/project"),
|
||||
ssh_keys)
|
||||
|
|
@ -427,7 +427,7 @@ class TestExecutorApi(ZooKeeperBaseTestCase):
|
|||
# Scheduler removes build request on completion
|
||||
client.remove(sched_a)
|
||||
|
||||
self.assertEqual(set(self.getZKTree('/zuul/executor')),
|
||||
self.assertEqual(set(self.getZKPaths('/zuul/executor')),
|
||||
set(['/zuul/executor/unzoned',
|
||||
'/zuul/executor/unzoned/locks',
|
||||
'/zuul/executor/unzoned/params',
|
||||
|
|
@ -685,12 +685,12 @@ class TestExecutorApi(ZooKeeperBaseTestCase):
|
|||
|
||||
class TestMergerApi(ZooKeeperBaseTestCase):
|
||||
def _assertEmptyRoots(self, client):
|
||||
self.assertEqual(self.getZKTree(client.REQUEST_ROOT), [])
|
||||
self.assertEqual(self.getZKTree(client.PARAM_ROOT), [])
|
||||
self.assertEqual(self.getZKTree(client.RESULT_ROOT), [])
|
||||
self.assertEqual(self.getZKTree(client.RESULT_DATA_ROOT), [])
|
||||
self.assertEqual(self.getZKTree(client.WAITER_ROOT), [])
|
||||
self.assertEqual(self.getZKTree(client.LOCK_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.REQUEST_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.PARAM_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.RESULT_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.RESULT_DATA_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.WAITER_ROOT), [])
|
||||
self.assertEqual(self.getZKPaths(client.LOCK_ROOT), [])
|
||||
self.assertEqual(self.getZKWatches(), {})
|
||||
|
||||
def test_merge_request(self):
|
||||
|
|
@ -837,12 +837,12 @@ class TestMergerApi(ZooKeeperBaseTestCase):
|
|||
result_data = {'result': 'ok'}
|
||||
server.reportResult(a, result_data)
|
||||
|
||||
self.assertEqual(set(self.getZKTree(client.RESULT_ROOT)),
|
||||
self.assertEqual(set(self.getZKPaths(client.RESULT_ROOT)),
|
||||
set(['/zuul/merger/results/A']))
|
||||
self.assertEqual(set(self.getZKTree(client.RESULT_DATA_ROOT)),
|
||||
self.assertEqual(set(self.getZKPaths(client.RESULT_DATA_ROOT)),
|
||||
set(['/zuul/merger/result-data/A',
|
||||
'/zuul/merger/result-data/A/0000000000']))
|
||||
self.assertEqual(self.getZKTree(client.WAITER_ROOT),
|
||||
self.assertEqual(self.getZKPaths(client.WAITER_ROOT),
|
||||
['/zuul/merger/waiters/A'])
|
||||
|
||||
# Merger removes and unlocks merge request on completion
|
||||
|
|
@ -936,12 +936,12 @@ class TestMergerApi(ZooKeeperBaseTestCase):
|
|||
server.remove(a)
|
||||
server.unlock(a)
|
||||
|
||||
self.assertEqual(set(self.getZKTree(client.RESULT_ROOT)),
|
||||
self.assertEqual(set(self.getZKPaths(client.RESULT_ROOT)),
|
||||
set(['/zuul/merger/results/A']))
|
||||
self.assertEqual(set(self.getZKTree(client.RESULT_DATA_ROOT)),
|
||||
self.assertEqual(set(self.getZKPaths(client.RESULT_DATA_ROOT)),
|
||||
set(['/zuul/merger/result-data/A',
|
||||
'/zuul/merger/result-data/A/0000000000']))
|
||||
self.assertEqual(self.getZKTree(client.WAITER_ROOT),
|
||||
self.assertEqual(self.getZKPaths(client.WAITER_ROOT),
|
||||
['/zuul/merger/waiters/A'])
|
||||
|
||||
# Scheduler "disconnects"
|
||||
|
|
|
|||
|
|
@ -16,9 +16,11 @@
|
|||
import argparse
|
||||
import babel.dates
|
||||
import datetime
|
||||
import json
|
||||
import jwt
|
||||
import logging
|
||||
import prettytable
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
|
@ -29,6 +31,8 @@ import urllib.parse
|
|||
import zuul.rpcclient
|
||||
import zuul.cmd
|
||||
from zuul.lib.config import get_default
|
||||
from zuul.zk import ZooKeeperClient
|
||||
from zuul.lib.keystorage import KeyStorage
|
||||
|
||||
|
||||
# todo This should probably live somewhere else
|
||||
|
|
@ -178,6 +182,7 @@ class Client(zuul.cmd.ZuulApp):
|
|||
description='valid commands',
|
||||
help='additional help')
|
||||
|
||||
# Autohold
|
||||
cmd_autohold = subparsers.add_parser(
|
||||
'autohold', help='hold nodes for failed job')
|
||||
cmd_autohold.add_argument('--tenant', help='tenant name',
|
||||
|
|
@ -221,6 +226,7 @@ class Client(zuul.cmd.ZuulApp):
|
|||
required=True)
|
||||
cmd_autohold_list.set_defaults(func=self.autohold_list)
|
||||
|
||||
# Enqueue/Dequeue
|
||||
cmd_enqueue = subparsers.add_parser('enqueue', help='enqueue a change')
|
||||
cmd_enqueue.add_argument('--tenant', help='tenant name',
|
||||
required=True)
|
||||
|
|
@ -277,6 +283,7 @@ class Client(zuul.cmd.ZuulApp):
|
|||
default=None)
|
||||
cmd_dequeue.set_defaults(func=self.dequeue)
|
||||
|
||||
# Promote
|
||||
cmd_promote = subparsers.add_parser('promote',
|
||||
help='promote one or more changes')
|
||||
cmd_promote.add_argument('--tenant', help='tenant name',
|
||||
|
|
@ -287,6 +294,7 @@ class Client(zuul.cmd.ZuulApp):
|
|||
required=True, nargs='+')
|
||||
cmd_promote.set_defaults(func=self.promote)
|
||||
|
||||
# Show
|
||||
cmd_show = subparsers.add_parser('show',
|
||||
help='show current statuses')
|
||||
cmd_show.set_defaults(func=self.show_running_jobs)
|
||||
|
|
@ -306,11 +314,13 @@ class Client(zuul.cmd.ZuulApp):
|
|||
# TODO: add filters such as queue, project, changeid etc
|
||||
show_running_jobs.set_defaults(func=self.show_running_jobs)
|
||||
|
||||
# Conf check
|
||||
cmd_conf_check = subparsers.add_parser(
|
||||
'tenant-conf-check',
|
||||
help='validate the tenant configuration')
|
||||
cmd_conf_check.set_defaults(func=self.validate)
|
||||
|
||||
# Auth token
|
||||
cmd_create_auth_token = subparsers.add_parser(
|
||||
'create-auth-token',
|
||||
help='create an Authentication Token for the web API',
|
||||
|
|
@ -349,6 +359,43 @@ class Client(zuul.cmd.ZuulApp):
|
|||
required=False)
|
||||
cmd_create_auth_token.set_defaults(func=self.create_auth_token)
|
||||
|
||||
# Key storage
|
||||
cmd_import_keys = subparsers.add_parser(
|
||||
'import-keys',
|
||||
help='import project keys to ZooKeeper',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
description=textwrap.dedent('''\
|
||||
Import previously exported project secret keys to ZooKeeper
|
||||
|
||||
Given a file with previously exported project keys, this
|
||||
command will import them into ZooKeeper. Existing keys
|
||||
will not be overwritten; to overwrite keys, add the
|
||||
--force flag.'''))
|
||||
cmd_import_keys.set_defaults(command='import-keys')
|
||||
cmd_import_keys.add_argument('path', type=str,
|
||||
help='key export file path')
|
||||
cmd_import_keys.add_argument('--force', action='store_true',
|
||||
help='overwrite existing keys')
|
||||
cmd_import_keys.set_defaults(func=self.import_keys)
|
||||
|
||||
cmd_export_keys = subparsers.add_parser(
|
||||
'export-keys',
|
||||
help='export project keys from ZooKeeper',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
description=textwrap.dedent('''\
|
||||
Export project secret keys from ZooKeeper
|
||||
|
||||
This command exports project secret keys from ZooKeeper
|
||||
and writes them to a file which is suitable for backing
|
||||
up and later use with the import-keys command.
|
||||
|
||||
The key contents are still protected by the keystore
|
||||
password and can not be used or decrypted without it.'''))
|
||||
cmd_export_keys.set_defaults(command='export-keys')
|
||||
cmd_export_keys.add_argument('path', type=str,
|
||||
help='key export file path')
|
||||
cmd_export_keys.set_defaults(func=self.export_keys)
|
||||
|
||||
return parser
|
||||
|
||||
def parseArguments(self, args=None):
|
||||
|
|
@ -741,6 +788,35 @@ class Client(zuul.cmd.ZuulApp):
|
|||
finally:
|
||||
sys.exit(err_code)
|
||||
|
||||
def export_keys(self):
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
zk_client = ZooKeeperClient.fromConfig(self.config)
|
||||
zk_client.connect()
|
||||
try:
|
||||
password = self.config["keystore"]["password"]
|
||||
except KeyError:
|
||||
raise RuntimeError("No key store password configured!")
|
||||
keystore = KeyStorage(zk_client, password=password)
|
||||
export = keystore.exportKeys()
|
||||
with open(os.open(self.args.path,
|
||||
os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
|
||||
json.dump(export, f)
|
||||
|
||||
def import_keys(self):
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
zk_client = ZooKeeperClient.fromConfig(self.config)
|
||||
zk_client.connect()
|
||||
try:
|
||||
password = self.config["keystore"]["password"]
|
||||
except KeyError:
|
||||
raise RuntimeError("No key store password configured!")
|
||||
keystore = KeyStorage(zk_client, password=password)
|
||||
with open(self.args.path, 'r') as f:
|
||||
import_data = json.load(f)
|
||||
keystore.importKeys(import_data, self.args.force)
|
||||
|
||||
|
||||
def main():
|
||||
Client().main()
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ from zuul.lib.config import get_default
|
|||
from zuul.lib.logutil import get_annotated_logger
|
||||
from zuul.lib.statsd import get_statsd
|
||||
from zuul.lib import filecomments
|
||||
from zuul.lib.keystorage import ZooKeeperKeyStorage
|
||||
from zuul.lib.keystorage import KeyStorage
|
||||
from zuul.lib.varnames import check_varnames
|
||||
|
||||
import zuul.lib.repl
|
||||
|
|
@ -3037,7 +3037,7 @@ class ExecutorServer(BaseMergeServer):
|
|||
|
||||
self.keep_jobdir = keep_jobdir
|
||||
self.jobdir_root = jobdir_root
|
||||
self.keystore = ZooKeeperKeyStorage(
|
||||
self.keystore = KeyStorage(
|
||||
self.zk_client,
|
||||
password=self._get_key_store_password())
|
||||
self._running = False
|
||||
|
|
|
|||
|
|
@ -12,12 +12,9 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import abc
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import cachetools
|
||||
|
|
@ -30,236 +27,8 @@ from zuul.zk import ZooKeeperBase
|
|||
RSA_KEY_SIZE = 2048
|
||||
|
||||
|
||||
class Migration(object):
|
||||
class KeyStorage(ZooKeeperBase):
|
||||
log = logging.getLogger("zuul.KeyStorage")
|
||||
version = 0
|
||||
parent = None
|
||||
|
||||
def verify(self, root):
|
||||
fn = os.path.join(root, '.version')
|
||||
if not os.path.exists(fn):
|
||||
return False
|
||||
with open(fn) as f:
|
||||
data = int(f.read().strip())
|
||||
if data == self.version:
|
||||
return True
|
||||
return False
|
||||
|
||||
def writeVersion(self, root):
|
||||
fn = os.path.join(root, '.version')
|
||||
with open(fn, 'w') as f:
|
||||
f.write(str(self.version))
|
||||
|
||||
def upgrade(self, root):
|
||||
pass
|
||||
|
||||
def verifyAndUpgrade(self, root):
|
||||
if self.verify(root):
|
||||
return
|
||||
if self.parent:
|
||||
self.parent.verifyAndUpgrade(root)
|
||||
self.log.info("Upgrading key storage to version %s" % self.version)
|
||||
self.upgrade(root)
|
||||
self.writeVersion(root)
|
||||
self.log.info("Finished upgrading key storage to version %s" %
|
||||
self.version)
|
||||
if not self.verify(root):
|
||||
raise Exception("Inconsistent result after migration")
|
||||
|
||||
|
||||
class MigrationV1(Migration):
|
||||
version = 1
|
||||
parent = None
|
||||
|
||||
"""Upgrade from the unversioned schema to version 1.
|
||||
|
||||
The original schema had secret keys in key_dir/connection/project.pem
|
||||
|
||||
This updates us to:
|
||||
key_dir/
|
||||
secrets/
|
||||
project/
|
||||
<connection>/
|
||||
<project>/
|
||||
<keyid>.pem
|
||||
ssh/
|
||||
project/
|
||||
<connection>/
|
||||
<project>/
|
||||
<keyid>.pem
|
||||
tenant/
|
||||
<tenant>/
|
||||
<keyid>.pem
|
||||
|
||||
Where keyids are integers to support future key rollover. In this
|
||||
case, they will all be 0.
|
||||
|
||||
"""
|
||||
|
||||
def upgrade(self, root):
|
||||
tmpdir = tempfile.mkdtemp(dir=root)
|
||||
tmpdirname = os.path.basename(tmpdir)
|
||||
connection_names = []
|
||||
for connection_name in os.listdir(root):
|
||||
if connection_name == tmpdirname:
|
||||
continue
|
||||
# Move existing connections out of the way (in case one of
|
||||
# them was called 'secrets' or 'ssh'.
|
||||
os.rename(os.path.join(root, connection_name),
|
||||
os.path.join(tmpdir, connection_name))
|
||||
connection_names.append(connection_name)
|
||||
os.makedirs(os.path.join(root, 'secrets', 'project'), 0o700)
|
||||
os.makedirs(os.path.join(root, 'ssh', 'project'), 0o700)
|
||||
os.makedirs(os.path.join(root, 'ssh', 'tenant'), 0o700)
|
||||
for connection_name in connection_names:
|
||||
connection_root = os.path.join(tmpdir, connection_name)
|
||||
for (dirpath, dirnames, filenames) in os.walk(connection_root):
|
||||
subdir = os.path.relpath(dirpath, connection_root)
|
||||
for fn in filenames:
|
||||
key_name = os.path.join(subdir, fn)
|
||||
project_name = key_name[:-len('.pem')]
|
||||
key_dir = os.path.join(root, 'secrets', 'project',
|
||||
connection_name, project_name)
|
||||
os.makedirs(key_dir, 0o700)
|
||||
old = os.path.join(tmpdir, connection_name, key_name)
|
||||
new = os.path.join(key_dir, '0.pem')
|
||||
self.log.debug("Moving key from %s to %s", old, new)
|
||||
os.rename(old, new)
|
||||
for (dirpath, dirnames, filenames) in os.walk(
|
||||
connection_root, topdown=False):
|
||||
os.rmdir(dirpath)
|
||||
os.rmdir(tmpdir)
|
||||
|
||||
|
||||
class KeyStorage(abc.ABC):
|
||||
log = logging.getLogger("zuul.KeyStorage")
|
||||
|
||||
@abc.abstractmethod
|
||||
def getProjectSSHKeys(self, connection_name, project_name):
|
||||
"""Return the private and public SSH keys for the project
|
||||
|
||||
A new key will be created if necessary.
|
||||
|
||||
:returns: A tuple containing the PEM encoded private key and
|
||||
base64 encoded public key.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def getProjectSecretsKeys(self, connection_name, project_name):
|
||||
"""Return the private and public secrets keys for the project
|
||||
|
||||
A new key will be created if necessary.
|
||||
|
||||
:returns: A tuple (private_key, public_key)
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class FileKeyStorage(KeyStorage):
|
||||
log = logging.getLogger("zuul.FileKeyStorage")
|
||||
current_version = MigrationV1
|
||||
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
migration = self.current_version()
|
||||
migration.verifyAndUpgrade(root)
|
||||
|
||||
def getProjectSecretsKeyFile(self, connection, project, version=None):
|
||||
"""Return the path to the private key used for the project's secrets"""
|
||||
# We don't actually support multiple versions yet
|
||||
if version is None:
|
||||
version = '0'
|
||||
return os.path.join(self.root, 'secrets', 'project',
|
||||
connection, project, version + '.pem')
|
||||
|
||||
def getProjectSSHKeyFile(self, connection, project, version=None):
|
||||
"""Return the path to the private ssh key for the project"""
|
||||
# We don't actually support multiple versions yet
|
||||
if version is None:
|
||||
version = '0'
|
||||
return os.path.join(self.root, 'ssh', 'project',
|
||||
connection, project, version + '.pem')
|
||||
|
||||
def hasProjectSSHKeys(self, connection_name, project_name):
|
||||
return os.path.isfile(
|
||||
self.getProjectSSHKeyFile(connection_name, project_name))
|
||||
|
||||
def setProjectSSHKey(self, connection_name, project_name, private_key):
|
||||
private_key_file = self.getProjectSSHKeyFile(connection_name,
|
||||
project_name)
|
||||
key_dir = os.path.dirname(private_key_file)
|
||||
if not os.path.isdir(key_dir):
|
||||
os.makedirs(key_dir, 0o700)
|
||||
|
||||
private_key.write_private_key_file(private_key_file)
|
||||
return private_key_file
|
||||
|
||||
def getProjectSSHKeys(self, connection_name, project_name):
|
||||
private_key_file = self._ensureSSHKeyFile(connection_name,
|
||||
project_name)
|
||||
|
||||
key = paramiko.RSAKey.from_private_key_file(private_key_file)
|
||||
with open(private_key_file, 'r') as f:
|
||||
private_key = f.read()
|
||||
public_key = key.get_base64()
|
||||
|
||||
return (private_key, 'ssh-rsa ' + public_key)
|
||||
|
||||
def _ensureSSHKeyFile(self, connection_name, project_name):
|
||||
private_key_file = self.getProjectSSHKeyFile(connection_name,
|
||||
project_name)
|
||||
if os.path.exists(private_key_file):
|
||||
return private_key_file
|
||||
|
||||
self.log.info("Generating SSH public key for project %s", project_name)
|
||||
pk = paramiko.RSAKey.generate(bits=RSA_KEY_SIZE)
|
||||
return self.setProjectSSHKey(connection_name, project_name, pk)
|
||||
|
||||
def hasProjectSecretsKeys(self, connection_name, project_name):
|
||||
return os.path.isfile(
|
||||
self.getProjectSecretsKeyFile(connection_name, project_name))
|
||||
|
||||
def setProjectSecretsKey(self, connection_name, project_name, private_key):
|
||||
filename = self.getProjectSecretsKeyFile(connection_name, project_name)
|
||||
key_dir = os.path.dirname(filename)
|
||||
if not os.path.isdir(key_dir):
|
||||
os.makedirs(key_dir, 0o700)
|
||||
|
||||
# Dump keys to filesystem. We only save the private key
|
||||
# because the public key can be constructed from it.
|
||||
self.log.info("Saving RSA keypair for project %s to %s",
|
||||
project_name, filename)
|
||||
|
||||
pem_private_key = encryption.serialize_rsa_private_key(private_key)
|
||||
# Ensure private key is read/write for zuul user only.
|
||||
with open(os.open(filename,
|
||||
os.O_CREAT | os.O_WRONLY, 0o600), 'wb') as f:
|
||||
f.write(pem_private_key)
|
||||
return filename
|
||||
|
||||
def getProjectSecretsKeys(self, connection_name, project_name):
|
||||
private_key_file = self._ensureSecretsKeyFile(connection_name,
|
||||
project_name)
|
||||
|
||||
# Load keypair
|
||||
with open(private_key_file, "rb") as f:
|
||||
return encryption.deserialize_rsa_keypair(f.read())
|
||||
|
||||
def _ensureSecretsKeyFile(self, connection_name, project_name):
|
||||
filename = self.getProjectSecretsKeyFile(connection_name, project_name)
|
||||
if os.path.isfile(filename):
|
||||
return filename
|
||||
|
||||
self.log.info("Generating RSA keypair for project %s", project_name)
|
||||
private_key, public_key = encryption.generate_rsa_keypair()
|
||||
|
||||
return self.setProjectSecretsKey(connection_name, project_name,
|
||||
private_key)
|
||||
|
||||
|
||||
class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
|
||||
log = logging.getLogger("zuul.ZooKeeperKeyStorage")
|
||||
SECRETS_PATH = "/keystorage/{}/{}/secrets"
|
||||
SSH_PATH = "/keystorage/{}/{}/ssh"
|
||||
|
||||
|
|
@ -267,31 +36,61 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
|
|||
super().__init__(zookeeper_client)
|
||||
self.password = password
|
||||
self.password_bytes = password.encode("utf-8")
|
||||
self.backup = backup
|
||||
|
||||
def _walk(self, root):
|
||||
ret = []
|
||||
children = self.kazoo_client.get_children(root)
|
||||
if children:
|
||||
for child in children:
|
||||
path = '/'.join([root, child])
|
||||
ret.extend(self._walk(path))
|
||||
else:
|
||||
data, _ = self.kazoo_client.get(root)
|
||||
try:
|
||||
ret.append((root, json.loads(data)))
|
||||
except Exception:
|
||||
self.log.error(f"Unable to load keys at {root}")
|
||||
# Keep processing exports
|
||||
return ret
|
||||
|
||||
def exportKeys(self):
|
||||
keys = {}
|
||||
for (path, data) in self._walk('/keystorage'):
|
||||
self.log.info(f"Exported: {path}")
|
||||
keys[path] = data
|
||||
return {'keys': keys}
|
||||
|
||||
def importKeys(self, import_data, overwrite):
|
||||
for path, data in import_data['keys'].items():
|
||||
if not path.startswith('/keystorage'):
|
||||
self.log.error(f"Invalid path: {path}")
|
||||
return
|
||||
data = json.dumps(data).encode('utf8')
|
||||
try:
|
||||
self.kazoo_client.create(path, value=data, makepath=True)
|
||||
self.log.info(f"Created key at {path}")
|
||||
except kazoo.exceptions.NodeExistsError:
|
||||
if overwrite:
|
||||
self.kazoo_client.set(path, value=data)
|
||||
self.log.info(f"Updated key at {path}")
|
||||
else:
|
||||
self.log.warning(f"Not overwriting existing key at {path}")
|
||||
|
||||
def getSSHKeysPath(self, connection_name, project_name):
|
||||
key_project_name = strings.unique_project_name(project_name)
|
||||
key_path = self.SSH_PATH.format(connection_name, key_project_name)
|
||||
return key_path
|
||||
|
||||
@cachetools.cached(cache={})
|
||||
def getProjectSSHKeys(self, connection_name, project_name):
|
||||
key_project_name = strings.unique_project_name(project_name)
|
||||
key_path = self.SSH_PATH.format(connection_name, key_project_name)
|
||||
key_path = self.getSSHKeysPath(connection_name, project_name)
|
||||
|
||||
try:
|
||||
key = self._getSSHKey(key_path)
|
||||
except kazoo.exceptions.NoNodeError:
|
||||
self.log.debug("Could not find existing SSH key for %s/%s",
|
||||
connection_name, project_name)
|
||||
if self.backup and self.backup.hasProjectSSHKeys(
|
||||
connection_name, project_name
|
||||
):
|
||||
self.log.info("Using SSH key for %s/%s from backup key store",
|
||||
connection_name, project_name)
|
||||
pk, _ = self.backup.getProjectSSHKeys(connection_name,
|
||||
project_name)
|
||||
with io.StringIO(pk) as o:
|
||||
key = paramiko.RSAKey.from_private_key(o)
|
||||
else:
|
||||
self.log.info("Generating a new SSH key for %s/%s",
|
||||
connection_name, project_name)
|
||||
key = paramiko.RSAKey.generate(bits=RSA_KEY_SIZE)
|
||||
self.log.info("Generating a new SSH key for %s/%s",
|
||||
connection_name, project_name)
|
||||
key = paramiko.RSAKey.generate(bits=RSA_KEY_SIZE)
|
||||
key_version = 0
|
||||
key_created = int(time.time())
|
||||
|
||||
|
|
@ -302,10 +101,6 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
|
|||
# creating the same SSH key.
|
||||
key = self._getSSHKey(key_path)
|
||||
|
||||
# Make sure the SSH key is also stored in the backup keystore
|
||||
if self.backup:
|
||||
self.backup.setProjectSSHKey(connection_name, project_name, key)
|
||||
|
||||
with io.StringIO() as o:
|
||||
key.write_private_key(o)
|
||||
private_key = o.getvalue()
|
||||
|
|
@ -337,32 +132,27 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
|
|||
data = json.dumps(keydata).encode("utf-8")
|
||||
self.kazoo_client.create(key_path, value=data, makepath=True)
|
||||
|
||||
@cachetools.cached(cache={})
|
||||
def getProjectSecretsKeys(self, connection_name, project_name):
|
||||
def getProjectSecretsKeysPath(self, connection_name, project_name):
|
||||
key_project_name = strings.unique_project_name(project_name)
|
||||
key_path = self.SECRETS_PATH.format(connection_name, key_project_name)
|
||||
return key_path
|
||||
|
||||
@cachetools.cached(cache={})
|
||||
def getProjectSecretsKeys(self, connection_name, project_name):
|
||||
key_path = self.getProjectSecretsKeysPath(
|
||||
connection_name, project_name)
|
||||
|
||||
try:
|
||||
pem_private_key = self._getSecretsKeys(key_path)
|
||||
except kazoo.exceptions.NoNodeError:
|
||||
self.log.debug("Could not find existing secrets key for %s/%s",
|
||||
connection_name, project_name)
|
||||
if self.backup and self.backup.hasProjectSecretsKeys(
|
||||
connection_name, project_name):
|
||||
self.log.info(
|
||||
"Using secrets key for %s/%s from backup key store",
|
||||
connection_name, project_name)
|
||||
private_key, public_key = self.backup.getProjectSecretsKeys(
|
||||
connection_name, project_name)
|
||||
else:
|
||||
self.log.info("Generating a new secrets key for %s/%s",
|
||||
connection_name, project_name)
|
||||
private_key, public_key = encryption.generate_rsa_keypair()
|
||||
|
||||
self.log.info("Generating a new secrets key for %s/%s",
|
||||
connection_name, project_name)
|
||||
private_key, public_key = encryption.generate_rsa_keypair()
|
||||
pem_private_key = encryption.serialize_rsa_private_key(
|
||||
private_key, self.password_bytes)
|
||||
key_version = 0
|
||||
key_created = int(time.time())
|
||||
|
||||
try:
|
||||
self._storeSecretsKeys(key_path, pem_private_key,
|
||||
key_version, key_created)
|
||||
|
|
@ -374,11 +164,6 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
|
|||
private_key, public_key = encryption.deserialize_rsa_keypair(
|
||||
pem_private_key, self.password_bytes)
|
||||
|
||||
# Make sure the private key is also stored in the backup keystore
|
||||
if self.backup:
|
||||
self.backup.setProjectSecretsKey(connection_name, project_name,
|
||||
private_key)
|
||||
|
||||
return private_key, public_key
|
||||
|
||||
def _getSecretsKeys(self, key_path):
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ from zuul.lib import commandsocket
|
|||
from zuul.lib.ansible import AnsibleManager
|
||||
from zuul.lib.config import get_default
|
||||
from zuul.lib.gear_utils import getGearmanFunctions
|
||||
from zuul.lib.keystorage import FileKeyStorage, ZooKeeperKeyStorage
|
||||
from zuul.lib.keystorage import KeyStorage
|
||||
from zuul.lib.logutil import get_annotated_logger
|
||||
from zuul.lib.queue import NamedQueue
|
||||
from zuul.lib.statsd import get_statsd, normalize_statsd_name
|
||||
|
|
@ -246,10 +246,9 @@ class Scheduler(threading.Thread):
|
|||
|
||||
def start(self):
|
||||
super(Scheduler, self).start()
|
||||
self.keystore = ZooKeeperKeyStorage(
|
||||
self.keystore = KeyStorage(
|
||||
self.zk_client,
|
||||
password=self._get_key_store_password(),
|
||||
backup=FileKeyStorage(self._get_key_dir()))
|
||||
password=self._get_key_store_password())
|
||||
|
||||
self._command_running = True
|
||||
self.log.debug("Starting command processor")
|
||||
|
|
@ -864,19 +863,6 @@ class Scheduler(threading.Thread):
|
|||
except KeyError:
|
||||
raise RuntimeError("No key store password configured!")
|
||||
|
||||
def _get_key_dir(self):
|
||||
state_dir = get_default(self.config, 'scheduler', 'state_dir',
|
||||
'/var/lib/zuul', expand_user=True)
|
||||
key_dir = os.path.join(state_dir, 'keys')
|
||||
if not os.path.exists(key_dir):
|
||||
os.mkdir(key_dir, 0o700)
|
||||
st = os.stat(key_dir)
|
||||
mode = st.st_mode & 0o777
|
||||
if mode != 0o700:
|
||||
raise Exception("Project key directory %s must be mode 0700; "
|
||||
"current mode is %o" % (key_dir, mode))
|
||||
return key_dir
|
||||
|
||||
def updateTenantLayout(self, tenant_name):
|
||||
self.log.debug("Updating layout of tenant %s", tenant_name)
|
||||
if self.unparsed_abide.ltime < self.system_config_cache.ltime:
|
||||
|
|
|
|||
Loading…
Reference in New Issue