Add commands to export/import keys to/from ZK
This removes the filesystem-based keystore in favor of only using ZooKeeper. Zuul will no longer load missing keys from the filesystem, nor will it write out decrypted copies of all keys to the filesystem. This is more secure since it allows sites better control over when and where secret data are written to disk. To provide for system backups to aid in disaster recovery in the case that the ZK data store is lost, two new scheduler commands are added: * export-keys * import-keys These write the password-protected versions of the keys (in fact, a raw dump of the ZK data) to the filesystem, and read the same data back in. An administrator can invoke export-keys before performing a system backup, and run import-keys to restore the data. A minor doc change recommending the use of ``zuul-scheduler stop`` was added as well, since that section is being updated. Change-Id: I5e6ea37c94ab73ec6f850591871c4127118414ed
This commit is contained in:
James E. Blair
parent
212d99bc7f
commit
0bf6e14720
|
|
@ -404,8 +404,18 @@ The following sections of ``zuul.conf`` are used by the scheduler:
|
|||
Operation
|
||||
~~~~~~~~~
|
||||
|
||||
To start the scheduler, run ``zuul-scheduler``. To stop it, kill the
|
||||
PID which was saved in the pidfile specified in the configuration.
|
||||
To start the scheduler, run ``zuul-scheduler``. To stop it, run
|
||||
``zuul-scheduler stop``.
|
||||
|
||||
Zuul stores private keys for each project it knows about in ZooKeeper.
|
||||
It is recommended that you periodically back up the private keys in
|
||||
case the ZooKeeper data store is lost or damaged. The scheduler
|
||||
provides two sub-commands for use in this case: ``zuul-scheduler
|
||||
export-keys`` and ``zuul-scheduler import-keys``. Each takes an
|
||||
argument to a filesystem path and will write the keys to, or read the
|
||||
keys from that path. The data in the exported files is still secured
|
||||
with the keystore passphrase, so be sure to retain it as well.
|
||||
|
||||
|
||||
Reconfiguration
|
||||
~~~~~~~~~~~~~~~
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
upgrade:
|
||||
- |
|
||||
Zuul no longer reads or writes project private key files from the
|
||||
scheduler's filesystem. In order to load existing keys into
|
||||
ZooKeeper, run version 4.6.0 of the scheduler at least once, if
|
||||
you haven't already.
|
||||
|
||||
A new command ``zuul-scheduler export-keys`` has been added to
|
||||
export the encrypted keys from ZooKeeper onto the filesystem for
|
||||
backup. Likewise, ``zuul-scheduler import-keys`` will load a
|
||||
previously-exported backup into ZooKeeper. It is recommended that
|
||||
you use these commands in system backup scripts.
|
||||
|
|
@ -109,6 +109,7 @@ import zuul.executor.client
|
|||
import zuul.lib.ansible
|
||||
import zuul.lib.connections
|
||||
import zuul.lib.auth
|
||||
import zuul.lib.keystorage
|
||||
import zuul.merger.client
|
||||
import zuul.merger.merger
|
||||
import zuul.merger.server
|
||||
|
|
@ -4134,6 +4135,17 @@ class BaseTestCase(testtools.TestCase):
|
|||
ChrootedKazooFixture(self.id())
|
||||
)
|
||||
|
||||
def getZKTree(self, path, ret=None):
|
||||
"""Return the contents of a ZK tree as a dictionary"""
|
||||
if ret is None:
|
||||
ret = {}
|
||||
for key in self.zk_client.client.get_children(path):
|
||||
subpath = os.path.join(path, key)
|
||||
ret[subpath] = self.zk_client.client.get(
|
||||
os.path.join(path, key))[0]
|
||||
self.getZKTree(subpath, ret)
|
||||
return ret
|
||||
|
||||
|
||||
class SymLink(object):
|
||||
def __init__(self, target):
|
||||
|
|
@ -4424,8 +4436,7 @@ class ZuulTestCase(BaseTestCase):
|
|||
'scheduler', 'command_socket',
|
||||
os.path.join(self.test_root, 'scheduler.socket'))
|
||||
if not self.config.has_option("keystore", "password"):
|
||||
self.config.set("keystore", "password",
|
||||
uuid.uuid4().hex)
|
||||
self.config.set("keystore", "password", 'keystorepassword')
|
||||
self.config.set('merger', 'git_dir', self.merger_src_root)
|
||||
self.config.set('executor', 'git_dir', self.executor_src_root)
|
||||
self.config.set('executor', 'private_key_file', self.private_key_file)
|
||||
|
|
@ -4494,6 +4505,9 @@ class ZuulTestCase(BaseTestCase):
|
|||
self.connection_event_queues = DefaultKeyDict(
|
||||
lambda cn: ConnectionEventQueue(self.zk_client, cn)
|
||||
)
|
||||
# requires zk client
|
||||
self.setupAllProjectKeys(self.config)
|
||||
|
||||
self.poller_events = {}
|
||||
self._configureSmtp()
|
||||
self._configureMqtt()
|
||||
|
|
@ -4641,7 +4655,6 @@ class ZuulTestCase(BaseTestCase):
|
|||
os.path.join(git_path, reponame))
|
||||
# Make test_root persist after ansible run for .flag test
|
||||
config.set('executor', 'trusted_rw_paths', self.test_root)
|
||||
self.setupAllProjectKeys(config)
|
||||
|
||||
return config
|
||||
|
||||
|
|
@ -4727,38 +4740,27 @@ class ZuulTestCase(BaseTestCase):
|
|||
def setupProjectKeys(self, source, project):
|
||||
# Make sure we set up an RSA key for the project so that we
|
||||
# don't spend time generating one:
|
||||
|
||||
if isinstance(project, dict):
|
||||
project = list(project.keys())[0]
|
||||
key_root = os.path.join(self.state_root, 'keys')
|
||||
if not os.path.isdir(key_root):
|
||||
os.mkdir(key_root, 0o700)
|
||||
fn = os.path.join(key_root, '.version')
|
||||
with open(fn, 'w') as f:
|
||||
f.write('1')
|
||||
# secrets key
|
||||
private_key_file = os.path.join(
|
||||
key_root, 'secrets', 'project', source, project, '0.pem')
|
||||
private_key_dir = os.path.dirname(private_key_file)
|
||||
self.log.debug("Installing test secrets keys for project %s at %s" % (
|
||||
project, private_key_file))
|
||||
if not os.path.isdir(private_key_dir):
|
||||
os.makedirs(private_key_dir)
|
||||
with open(os.path.join(FIXTURE_DIR, 'private.pem')) as i:
|
||||
with open(private_key_file, 'w') as o:
|
||||
o.write(i.read())
|
||||
|
||||
password = self.config.get("keystore", "password")
|
||||
keystore = zuul.lib.keystorage.KeyStorage(
|
||||
self.zk_client, password=password)
|
||||
import_list = []
|
||||
|
||||
path = keystore.getProjectSecretsKeysPath(source, project)
|
||||
# Strip /keystore, just like the export routine does
|
||||
path = path.split('/', 2)[2]
|
||||
with open(os.path.join(FIXTURE_DIR, 'secrets.json'), 'rb') as i:
|
||||
import_list.append((path, i.read()))
|
||||
|
||||
# ssh key
|
||||
private_key_file = os.path.join(
|
||||
key_root, 'ssh', 'project', source, project, '0.pem')
|
||||
private_key_dir = os.path.dirname(private_key_file)
|
||||
self.log.debug("Installing test ssh keys for project %s at %s" % (
|
||||
project, private_key_file))
|
||||
if not os.path.isdir(private_key_dir):
|
||||
os.makedirs(private_key_dir)
|
||||
with open(os.path.join(FIXTURE_DIR, 'ssh.pem')) as i:
|
||||
with open(private_key_file, 'w') as o:
|
||||
o.write(i.read())
|
||||
path = keystore.getSSHKeysPath(source, project)
|
||||
path = path.split('/', 2)[2]
|
||||
with open(os.path.join(FIXTURE_DIR, 'ssh.json'), 'rb') as i:
|
||||
import_list.append((path, i.read()))
|
||||
|
||||
keystore.importKeys(import_list)
|
||||
|
||||
def getCurrentLtime(self):
|
||||
"""Get the logical timestamp as seen by the Zookeeper cluster."""
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
{"schema": 1, "keys": [{"version": 0, "created": 1626909706, "private_key": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,8448A7DF16A5CF040EA107EB6B0A5235\n\ncLcAbSpqh3pnH5Q8ATw47yPmjAoPbDwq52mxDx53PmD0DfYw6u197jhNxRZmedu4\nvRk226Gr4ppsvj62LKK6Na95nLl+S9pYHKv6ZI4gVx8sliQDthgpWbYt/gYE9Wk2\n/wcZ58iQJTFzjM0LrhOgx4HXuuohKhlIjDj/sdo5cJiva4wNyxJlgEcjbMsLBlDG\no2Li7Wn9wC/5wKrG184yHbJJhbTV+FtGJeMcJLfy0l6Kod6jnYi1p5O+ZlMCBnqC\nF1VjqjeF95uvO075V33DFsdFdDWpSWHOQvLmzX2V0izNqW+06jurkS948hBYnJRK\ntL6waUzjLOoU8fEPfEoxVOCJssEsqgSgezE0YL4w7l/VSyvO+G/AS0uYcpCMT4dV\n9rZJ0WGwKgvfpVysR0hREBQ45zxuxo3Qh74GJKM10c2j1FPBk/M2YUhje54yHJ4a\nvWrmIXQLCQ9PtiMdWKaijIhPce0l8pRSomYy4kwZFzhLlzKCd1BzLpT4TwIm1P0H\nToyZu4zIT9fGdfX+WpZRbUnvIFVSmTpMJ47Yp21ScVJGN5ezgNfe8Eih2rgaAB0C\nZ1bXtYJxPkYmWPZn8llk80d5uLKDHJDMpL8kge+Km1MyTPayZRVknG4z160ukTTv\nv/nLfMmzeD9BuGFPSAMjNEntc0ld/ntVntvysLP3NLJguu4T0QTvbMJjRPfosKIv\n58PLCb9eQHAArSdkqV1ybMqpFATGdnbTg5uHodpGAoDZp7l8cfgO43fUTleIdRri\nVeNdnH4iSedcZ/SZBy13xmC7MUEwotToAHxHPiKvoonvq1gv4YWZyMio50ySKR5u\neJmEQZ2+Veiqk3a6ldxhjCuAx6E4iwOZWsASA6GhfhZNOv/oUqAjguLKS6bzCuBn\nnfp9MHHdsf+FfJJ1/OXYmvnVz9UB3bQna/+e/KJd4Ntj1D8IJj8OMsT3Kjsz3yJ9\n3CVJYnfAH1sS/7qMyv1hDTcDOy7EwbHc3LznKo/0PUuO9Rvm+eoyzM16XAvOW4Zu\nKALuC98FUtUKO1TpuZU59w5NkwCoWMnulVgYPjRNOymPUgFpiopcbCQmRJRDJ0hp\n5syncDp5XGiq51TXZ9nXQRS2c0TNIHlR+hT3NvJ+H/hO7ZjmiA7uVsdRubhJrQsv\nqZMACSkhY4etnDWVXW7G90eBBHeT6hobVwSHobaxkYkMmPD2Nrmsoh9a6y3Kpd8S\n0BVdQA6vNig85B3bP640otL9jIN52tLAoA//CvFkiIeZunBIWqfEymlhe06j0LGP\ntDtvF+3vqoyIGod0kCj47czEe7f/vhDyzAjHKU13vqrUdVPOdiAUUPMvpppdt0Iy\nODMFRIZqEOZXFTu08YiB22O32wFWy8rPvPSUcva+1dV6VKW0m51BkBgDeczqearn\nNjMyh1TA2KayeZXj+CaACUfgrgtgwZ2XwbZLHudRbCTOpGtrVtCQmdgE9qHFaFED\nW22MHkQ3dR8NPx9XwQIGRMYS6SavrpPsgI5J4EnQ3F8nxcBK7vWWcL5cBCixPFVz\n30bDcsx8HUoQUzjJEqMwZ/vnuElgBfYpjYlAP3hWZp/BGQr3dlUisG1c6bfhoefq\nNNl3Pubt1tj72I6V43OqUAF1zxsMiXLmRR0Eqcyd2VA2SZ+n3gD5aEmJ2pfGNESW\npgRSNA/BuB2ToRTYWo1yW//FNSmuNPwYPqB4rcG4F7m3/W3cV6olXCL6pFHKKjva\nnNU+Gn2TCoVTbmSRab9EXgO5TbUybFU/RV4OAgOhkIFe32Tg7ksp/LYBuytQ8kAG\n19ve5b5xC7LhEbEJEl435eU8coAavPW9+BMGuQUlDkg7Kq3N29HeDZ+OpAy4FJDu\nwkNb68zno1f7F71ZgkHH/AKRwvq5ut+TFA91Vk8e+w+N6ftszjJFs7917TENLKW/\n0+eAlYFY+bHmqxjJbEnfCMNMBMz5H7lHsOZmz+TGL8+DTPzj0t+PoSO15xi4Ga7F\n1X3UQQxQkA42nqGxhVMY7SE/wTRwT02ZUKPEus1IsiA73uZNJst3q5ddwzt8DkfU\n/Ov2m6M/PDnwn26tZQnNr1GK6jHMtnTa6xQDeXHUMymxbRc6jE71cPuHVEZgsfJo\n54f1vrInRDDPb2gjydRtxDIk3Fd5apBLZymadSqqnm6G4LLzCVQ8TSA6Rrya3J3V\nQ1gD9wEwhRmwtheNAV/qdBk3KCYAJAfnQAANuT87oyeJAEiPRKSSIACd9zkS/tq1\nptjeFb4y+Gs2x5qPzGUHICvBteuS6h/kdNMGBzcwdntXPV1hM4lafnCj9VCMCF1k\nwPz/M740VY6zdxo8d2LYuxKpUYcTzkZbuJorKGVgW1I4EkfAXF2BToAQ8kKsKVsK\nuDOo+s8ivAswlAX4gCtvVnESqTALwmFbJsq8OeleT4tSiNeIwhZM0sh3VdztD0Jw\nAJOULbcN08wYilFb+bgvzDlEIauiEv/8agmyTf3j+7q9FpPPzGAMrQbLZ1yhx8IN\n9u+ks9Xp3vtctJXMKJVyUVUYFGEEK8hbHNazLmpo2N7pV90uWlKfPGo9iF9osd1V\nwERF/INtO9gyPtsQNMylbvcVouAEx+A/q3+UtUIQyuc9t6+RWlxpVgFDTjylJsuI\nP3NYxt7kM9OP8TPTHoooQd28EOIVgzsXS8OmtKgTxScaU09/6EaAFPjWwVZJaA0E\nqBXlS7SkucEw7YU5Qx65y0B+r/keIdu7Cvc3XXyeISKZTzo94VIJvCC/cTIMZY/K\n9vnPJxnwmj0EnHB8Wb2j8FZqEIpDwh96mTWoroX4PGZdnKaaxL1vQISlss4KQ3y4\nJGVRyauK5rC9SwBWBCiZCPEHZ5q82m8IT3w41/umY4S01a33D1pgTa1rZEj38eVU\n6n9dP67kjP1C7m3YIDxNSX9+yv1sx8PBZ6ixomj/KAeXNbWzveDwPJM4H09qyvKU\nAhVPwIdUuJMbXsjumjxXXSOw2+A8KwAyPQ8vry3lshSP7QvtDOkCY33G3iOJJoQ1\nyv8Y8Hw6GPVrzGnjkYKRNfwlWirfad6e9sHGxv2VrH4bZzulXwnobZ5FpXJUv8s7\ndICGLSBrxozAoZiMLVyp3MMeo/iudcIIPb423R8VNiAiMxLpMezqbYEnZezzRyJo\n-----END RSA PRIVATE KEY-----\n"}]}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"schema": 1, "keys": [{"version": 0, "created": 1626909706, "private_key": "-----BEGIN RSA PRIVATE KEY-----\nProc-Type: 4,ENCRYPTED\nDEK-Info: AES-256-CBC,7FF563CF60708DF4F2999DF100EC742D\n\nqecR3BS3nKFoKuexputN2ubp1nfHwFWw6VvD4wb3uBXxgzUeaSP9qK7qEUJU6bU9\nPvTJTPLrnNTD0zwsVyw2uJEv3PTFZ7tCVEocUNP9LoGrta8xXZ3QGDzo3rJ6GPCK\nR4UsNZa4KStnqAtcxw5nhhGXPZT0FDBsT9dgSi6JeYZZIifojWZG+jaoSXEaRe+O\nF34ys73gsmWo9tOvW8mN+YuFTKc8TlCviIa+NnqDHotF7tNp8SxIPJqBE4lygiEo\nzuHKJOAlohvLqVQsKk1383w5Xj8nKO33EhsAidqy4VBwnu/e4pbMIgdBkCFqHiVb\nTIhb452LMpaZr/fVge8qOXr/oULLXkGppVVxza0M+9fnYIwjEDryrFy0JTk88JIg\n3tw9CSn6DEN8yDXtkZZnOvcbJKrlOZll3e1A7jJpVAALM/NdVbGqVq2q9bWCf+Ql\nCHY0g9DAd//0/wgl2IRQ1jfOPMIJATsgrj7h6g4vj/Fsv9vKO9LPO9wILAlHHEc9\nYd0rBDT7nLTgdRNghf+auRBvi8fZxfbGXuv86Ei1A9iS/bf6+SlVt8V8tHd9/mnx\nq7PHlwUXg3IPONn0U+o6/uIJKW+t55PdmhXUhPc1TbvVH3QbiywzzR+q+lw5QhgO\nqJqKiJhEsfSuAk5B2/xBQxQpuFGs+8MH/AaapUu8rD+skZphWnxZZLmVYHCG111o\ny36RmhSBXkWy8wwpM9VXyDAJ9WhjE0VpFX5/ZsST+UwZIeGmL6NU4NQ9jYS2KAGj\nfDRlU3O4iA92kCYyj/tNeetkj+pDO6otYdUSwbblE8jyCqulV2IYCabDe+cfauym\nTMaBdFNRw9zB0poqpNqbHflGihuoN7Gb37ST7ILNsqPLqIP52oRS/JHHeVuaiQsJ\naUM0L+rOOxRImkEukp0IS7I8X5m+KJ8tRe5s1i3ojBykGpCtodxLGcxEebM9ncC+\n3U2m4nsyBMeiqVd2DJgwi7Qep4YTUXwzSxzGxLNTXQfokN598pi9RTu0AcTf0/Rs\n4a7sOavbuRJMEAzvoDDHOAfm39qVP4HwJ0142Cg9Y0P6VUgJSGXSagsRwPIFSOpJ\nnanC8sANMG4zSK4Aujyf4S/roK4uK3p5nkmF+sn/bD7XD4ACYBwBos+cQxOBRkiZ\nOxEevbP2NWlkMFU8P8HhVJQyv8YvV+K2tWYE3TS/v/DUhE22sObepzwkE/69sZWb\nBTEMsZEIaC1nH0JDLJ7PQqRgcRWu8gzsTUQZtd9VPypenE1XmJp/9jPqAAa5Fa3v\nZPnUPDnykUBaiLKINb4daMcf4MMJ4F1cHB8hbGKkDqNoUIa0U8MT/t8aeSWquSyk\nCKHbIqDHJESn/04YPFrSEx/36bP77DiiA7SKI/YjqSF7h+QXGwAiO7P2FBD05YST\nkQtDwr2vPH2w2PK2aqfacedZREYROWh4XzCYuLjFd7dNQ8QhKkxtcX2RxR6EJHAf\nYgaCyz2Il9jkR+7e3uVNDwv5aUIPqa/+PiNQzhNR9nrP513cxuWbdZlIemft11MO\nwY53ubT/oiRK1K+5/ZNZUt5REVpLr1Kls8nrOACL2RlfFrtizmRslIOCGIDIIg9M\n-----END RSA PRIVATE KEY-----\n"}]}
|
||||
|
|
@ -12,10 +12,6 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
import fixtures
|
||||
from unittest.mock import patch
|
||||
|
||||
from zuul.lib import encryption
|
||||
from zuul.lib import keystorage
|
||||
from zuul.zk import ZooKeeperClient
|
||||
|
|
@ -23,55 +19,7 @@ from zuul.zk import ZooKeeperClient
|
|||
from tests.base import BaseTestCase
|
||||
|
||||
|
||||
class TestFileKeyStorage(BaseTestCase):
|
||||
|
||||
def _setup_keys(self, root, connection_name, project_name):
|
||||
cn = os.path.join(root, connection_name)
|
||||
if '/' in project_name:
|
||||
pn = os.path.join(cn, os.path.dirname(project_name))
|
||||
os.makedirs(pn)
|
||||
fn = os.path.join(cn, project_name + '.pem')
|
||||
with open(fn, 'w'):
|
||||
pass
|
||||
|
||||
def assertFile(self, root, path, contents=None):
|
||||
fn = os.path.join(root, path)
|
||||
self.assertTrue(os.path.exists(fn))
|
||||
if contents:
|
||||
with open(fn) as f:
|
||||
self.assertEqual(contents, f.read())
|
||||
|
||||
def assertPaths(self, root, paths):
|
||||
seen = set()
|
||||
for dirpath, dirnames, filenames in os.walk(root):
|
||||
for d in dirnames:
|
||||
seen.add(os.path.join(dirpath[len(root) + 1:], d))
|
||||
for f in filenames:
|
||||
seen.add(os.path.join(dirpath[len(root) + 1:], f))
|
||||
self.assertEqual(set(paths), seen)
|
||||
|
||||
def test_key_storage(self):
|
||||
root = self.useFixture(fixtures.TempDir()).path
|
||||
self._setup_keys(root, 'gerrit', 'org/example')
|
||||
keystorage.FileKeyStorage(root)
|
||||
self.assertFile(root, '.version', '1')
|
||||
self.assertPaths(root, [
|
||||
'.version',
|
||||
'secrets',
|
||||
'secrets/project',
|
||||
'secrets/project/gerrit',
|
||||
'secrets/project/gerrit/org',
|
||||
'secrets/project/gerrit/org/example',
|
||||
'secrets/project/gerrit/org/example/0.pem',
|
||||
'ssh',
|
||||
'ssh/project',
|
||||
'ssh/tenant',
|
||||
])
|
||||
# It shouldn't need to upgrade this time
|
||||
keystorage.FileKeyStorage(root)
|
||||
|
||||
|
||||
class TestZooKeeperKeyStorage(BaseTestCase):
|
||||
class TestKeyStorage(BaseTestCase):
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
|
@ -85,90 +33,8 @@ class TestZooKeeperKeyStorage(BaseTestCase):
|
|||
self.addCleanup(self.zk_client.disconnect)
|
||||
self.zk_client.connect()
|
||||
|
||||
def test_backup(self):
|
||||
root = self.useFixture(fixtures.TempDir()).path
|
||||
backup = keystorage.FileKeyStorage(root)
|
||||
key_store = keystorage.ZooKeeperKeyStorage(
|
||||
self.zk_client, password="DEADBEEF", backup=backup)
|
||||
|
||||
# Create keys in the backup keystore
|
||||
backup_secrets_pk = encryption.serialize_rsa_private_key(
|
||||
backup.getProjectSecretsKeys("github", "org/project")[0])
|
||||
backup_ssh_keys = backup.getProjectSSHKeys("github", "org/project")
|
||||
|
||||
self.assertEqual(
|
||||
encryption.serialize_rsa_private_key(
|
||||
key_store.getProjectSecretsKeys("github", "org/project")[0]
|
||||
), backup_secrets_pk)
|
||||
self.assertEqual(
|
||||
key_store.getProjectSSHKeys("github", "org/project"),
|
||||
backup_ssh_keys)
|
||||
|
||||
# Keys should initially not be in the backup keystore
|
||||
self.assertFalse(
|
||||
backup.hasProjectSecretsKeys("github", "org/project1"))
|
||||
self.assertFalse(
|
||||
backup.hasProjectSSHKeys("github", "org/project1"))
|
||||
|
||||
self.assertIsNotNone(
|
||||
key_store.getProjectSecretsKeys("github", "org/project1"))
|
||||
self.assertIsNotNone(
|
||||
key_store.getProjectSSHKeys("github", "org/project1"))
|
||||
|
||||
# Keys should now also exist in the backup keystore
|
||||
self.assertTrue(
|
||||
backup.hasProjectSecretsKeys("github", "org/project1"))
|
||||
self.assertTrue(
|
||||
backup.hasProjectSSHKeys("github", "org/project1"))
|
||||
|
||||
def test_key_store_upgrade(self):
|
||||
# Test that moving an unencrypted key on the file system to ZK
|
||||
# (encrypted) works as expected and the backup keys stay the same.
|
||||
root = self.useFixture(fixtures.TempDir()).path
|
||||
backup = keystorage.FileKeyStorage(root)
|
||||
key_store = keystorage.ZooKeeperKeyStorage(
|
||||
self.zk_client, password="DECAFBAD", backup=backup)
|
||||
|
||||
# Create reference keys in the backup key store
|
||||
ref_secrets_pk = encryption.serialize_rsa_private_key(
|
||||
backup.getProjectSecretsKeys("github", "org/project")[0])
|
||||
ref_ssh_keys = backup.getProjectSSHKeys("github", "org/project")
|
||||
|
||||
# Make sure we get the backup keys via the primary key store.
|
||||
self.assertEqual(
|
||||
encryption.serialize_rsa_private_key(
|
||||
key_store.getProjectSecretsKeys("github", "org/project")[0]
|
||||
), ref_secrets_pk)
|
||||
self.assertEqual(key_store.getProjectSSHKeys("github", "org/project"),
|
||||
ref_ssh_keys)
|
||||
|
||||
# Make sure we can still read the updates keys from the backup
|
||||
# key store.
|
||||
self.assertEqual(
|
||||
encryption.serialize_rsa_private_key(
|
||||
backup.getProjectSecretsKeys("github", "org/project")[0]
|
||||
), ref_secrets_pk)
|
||||
self.assertEqual(backup.getProjectSSHKeys("github", "org/project"),
|
||||
ref_ssh_keys)
|
||||
|
||||
# Make sure that the backup key store is not used after keys have been
|
||||
# written to the primary key store.
|
||||
exc = AssertionError("Keys should not be loaded from backup store")
|
||||
# with patch.object(backup, "getProjectSecretsKeys", side_effect=exc):
|
||||
with patch.object(backup, "getProjectSecretsKeys", side_effect=exc):
|
||||
self.assertEqual(
|
||||
encryption.serialize_rsa_private_key(
|
||||
key_store.getProjectSecretsKeys("github", "org/project")[0]
|
||||
), ref_secrets_pk)
|
||||
|
||||
exc = AssertionError("SSH keys should not be loaded from backup store")
|
||||
with patch.object(backup, "getProjectSSHKeys", side_effect=exc):
|
||||
self.assertEqual(
|
||||
key_store.getProjectSSHKeys("github", "org/project"),
|
||||
ref_ssh_keys)
|
||||
|
||||
def test_without_backup(self):
|
||||
key_store = keystorage.ZooKeeperKeyStorage(
|
||||
key_store = keystorage.KeyStorage(
|
||||
self.zk_client, password="DECAFBAD")
|
||||
secrets_pk = encryption.serialize_rsa_private_key(
|
||||
key_store.getProjectSecretsKeys("github", "org/project")[0])
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ import os
|
|||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import textwrap
|
||||
import threading
|
||||
import time
|
||||
|
|
@ -9002,3 +9004,46 @@ class TestEventProcessing(ZuulTestCase):
|
|||
dict(name='tagjob', result='SUCCESS'),
|
||||
dict(name='checkjob', result='SUCCESS', changes='1,1'),
|
||||
], ordered=False)
|
||||
|
||||
|
||||
class TestKeyExportImport(ZuulTestCase):
|
||||
tenant_config_file = 'config/single-tenant/main.yaml'
|
||||
|
||||
def test_export_import(self):
|
||||
# Test a round trip export/import of keys
|
||||
export_root = os.path.join(self.test_root, 'export')
|
||||
config_file = os.path.join(self.test_root, 'zuul.conf')
|
||||
with open(config_file, 'w') as f:
|
||||
self.config.write(f)
|
||||
|
||||
# Save a copy of the keys in ZK
|
||||
old_data = self.getZKTree('/keystorage')
|
||||
|
||||
# Export keys
|
||||
p = subprocess.Popen(
|
||||
[os.path.join(sys.prefix, 'bin/zuul-scheduler'),
|
||||
'-c', config_file,
|
||||
'export-keys', export_root],
|
||||
stdout=subprocess.PIPE)
|
||||
out, _ = p.communicate()
|
||||
self.log.debug(out.decode('utf8'))
|
||||
|
||||
# Delete keys from ZK
|
||||
self.zk_client.client.delete('/keystorage', recursive=True)
|
||||
|
||||
# Make sure it's really gone
|
||||
with testtools.ExpectedException(NoNodeError):
|
||||
self.getZKTree('/keystorage')
|
||||
|
||||
# Import keys
|
||||
p = subprocess.Popen(
|
||||
[os.path.join(sys.prefix, 'bin/zuul-scheduler'),
|
||||
'-c', config_file,
|
||||
'import-keys', export_root],
|
||||
stdout=subprocess.PIPE)
|
||||
out, _ = p.communicate()
|
||||
self.log.debug(out.decode('utf8'))
|
||||
|
||||
# Make sure the new data matches the original
|
||||
new_data = self.getZKTree('/keystorage')
|
||||
self.assertEqual(new_data, old_data)
|
||||
|
|
|
|||
|
|
@ -42,9 +42,23 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
|
|||
'listed, all tenants will be validated. '
|
||||
'Note: this requires the gearman server and '
|
||||
'will distribute work to mergers.')
|
||||
parser.add_argument('command',
|
||||
choices=zuul.scheduler.COMMANDS,
|
||||
nargs='?')
|
||||
subparser = parser.add_subparsers(help='sub-command help')
|
||||
parser.set_defaults(command=None)
|
||||
for command in zuul.scheduler.COMMANDS:
|
||||
p = subparser.add_parser(command)
|
||||
p.set_defaults(command=command)
|
||||
import_parser = subparser.add_parser(
|
||||
'import-keys',
|
||||
help='import project keys to ZooKeeper')
|
||||
import_parser.set_defaults(command='import-keys')
|
||||
import_parser.add_argument('path', type=str,
|
||||
help='filesystem root to read keys')
|
||||
export_parser = subparser.add_parser(
|
||||
'export-keys',
|
||||
help='export project keys from ZooKeeper')
|
||||
export_parser.set_defaults(command='export-keys')
|
||||
export_parser.add_argument('path', type=str,
|
||||
help='filesystem root to write keys')
|
||||
return parser
|
||||
|
||||
def parseArguments(self, args=None):
|
||||
|
|
@ -124,6 +138,11 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
|
|||
os.kill(self.gear_server_pid, signal.SIGKILL)
|
||||
|
||||
def run(self):
|
||||
if self.args.command == 'export-keys':
|
||||
return zuul.scheduler.export_keys(self.config, self.args.path)
|
||||
if self.args.command == 'import-keys':
|
||||
return zuul.scheduler.import_keys(self.config, self.args.path)
|
||||
|
||||
if self.args.command in zuul.scheduler.COMMANDS:
|
||||
self.send_command(self.args.command)
|
||||
sys.exit(0)
|
||||
|
|
|
|||
|
|
@ -41,7 +41,7 @@ from zuul.lib.config import get_default
|
|||
from zuul.lib.logutil import get_annotated_logger
|
||||
from zuul.lib.statsd import get_statsd
|
||||
from zuul.lib import filecomments
|
||||
from zuul.lib.keystorage import ZooKeeperKeyStorage
|
||||
from zuul.lib.keystorage import KeyStorage
|
||||
from zuul.lib.varnames import check_varnames
|
||||
|
||||
import gear
|
||||
|
|
@ -3052,7 +3052,7 @@ class ExecutorServer(BaseMergeServer):
|
|||
|
||||
self.keep_jobdir = keep_jobdir
|
||||
self.jobdir_root = jobdir_root
|
||||
self.keystore = ZooKeeperKeyStorage(
|
||||
self.keystore = KeyStorage(
|
||||
self.zk_client,
|
||||
password=self._get_key_store_password())
|
||||
self._running = False
|
||||
|
|
|
|||
|
|
@ -12,12 +12,9 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import abc
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
import cachetools
|
||||
|
|
@ -30,236 +27,8 @@ from zuul.zk import ZooKeeperBase
|
|||
RSA_KEY_SIZE = 2048
|
||||
|
||||
|
||||
class Migration(object):
|
||||
class KeyStorage(ZooKeeperBase):
|
||||
log = logging.getLogger("zuul.KeyStorage")
|
||||
version = 0
|
||||
parent = None
|
||||
|
||||
def verify(self, root):
|
||||
fn = os.path.join(root, '.version')
|
||||
if not os.path.exists(fn):
|
||||
return False
|
||||
with open(fn) as f:
|
||||
data = int(f.read().strip())
|
||||
if data == self.version:
|
||||
return True
|
||||
return False
|
||||
|
||||
def writeVersion(self, root):
|
||||
fn = os.path.join(root, '.version')
|
||||
with open(fn, 'w') as f:
|
||||
f.write(str(self.version))
|
||||
|
||||
def upgrade(self, root):
|
||||
pass
|
||||
|
||||
def verifyAndUpgrade(self, root):
|
||||
if self.verify(root):
|
||||
return
|
||||
if self.parent:
|
||||
self.parent.verifyAndUpgrade(root)
|
||||
self.log.info("Upgrading key storage to version %s" % self.version)
|
||||
self.upgrade(root)
|
||||
self.writeVersion(root)
|
||||
self.log.info("Finished upgrading key storage to version %s" %
|
||||
self.version)
|
||||
if not self.verify(root):
|
||||
raise Exception("Inconsistent result after migration")
|
||||
|
||||
|
||||
class MigrationV1(Migration):
|
||||
version = 1
|
||||
parent = None
|
||||
|
||||
"""Upgrade from the unversioned schema to version 1.
|
||||
|
||||
The original schema had secret keys in key_dir/connection/project.pem
|
||||
|
||||
This updates us to:
|
||||
key_dir/
|
||||
secrets/
|
||||
project/
|
||||
<connection>/
|
||||
<project>/
|
||||
<keyid>.pem
|
||||
ssh/
|
||||
project/
|
||||
<connection>/
|
||||
<project>/
|
||||
<keyid>.pem
|
||||
tenant/
|
||||
<tenant>/
|
||||
<keyid>.pem
|
||||
|
||||
Where keyids are integers to support future key rollover. In this
|
||||
case, they will all be 0.
|
||||
|
||||
"""
|
||||
|
||||
def upgrade(self, root):
|
||||
tmpdir = tempfile.mkdtemp(dir=root)
|
||||
tmpdirname = os.path.basename(tmpdir)
|
||||
connection_names = []
|
||||
for connection_name in os.listdir(root):
|
||||
if connection_name == tmpdirname:
|
||||
continue
|
||||
# Move existing connections out of the way (in case one of
|
||||
# them was called 'secrets' or 'ssh'.
|
||||
os.rename(os.path.join(root, connection_name),
|
||||
os.path.join(tmpdir, connection_name))
|
||||
connection_names.append(connection_name)
|
||||
os.makedirs(os.path.join(root, 'secrets', 'project'), 0o700)
|
||||
os.makedirs(os.path.join(root, 'ssh', 'project'), 0o700)
|
||||
os.makedirs(os.path.join(root, 'ssh', 'tenant'), 0o700)
|
||||
for connection_name in connection_names:
|
||||
connection_root = os.path.join(tmpdir, connection_name)
|
||||
for (dirpath, dirnames, filenames) in os.walk(connection_root):
|
||||
subdir = os.path.relpath(dirpath, connection_root)
|
||||
for fn in filenames:
|
||||
key_name = os.path.join(subdir, fn)
|
||||
project_name = key_name[:-len('.pem')]
|
||||
key_dir = os.path.join(root, 'secrets', 'project',
|
||||
connection_name, project_name)
|
||||
os.makedirs(key_dir, 0o700)
|
||||
old = os.path.join(tmpdir, connection_name, key_name)
|
||||
new = os.path.join(key_dir, '0.pem')
|
||||
self.log.debug("Moving key from %s to %s", old, new)
|
||||
os.rename(old, new)
|
||||
for (dirpath, dirnames, filenames) in os.walk(
|
||||
connection_root, topdown=False):
|
||||
os.rmdir(dirpath)
|
||||
os.rmdir(tmpdir)
|
||||
|
||||
|
||||
class KeyStorage(abc.ABC):
|
||||
log = logging.getLogger("zuul.KeyStorage")
|
||||
|
||||
@abc.abstractmethod
|
||||
def getProjectSSHKeys(self, connection_name, project_name):
|
||||
"""Return the private and public SSH keys for the project
|
||||
|
||||
A new key will be created if necessary.
|
||||
|
||||
:returns: A tuple containing the PEM encoded private key and
|
||||
base64 encoded public key.
|
||||
"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
def getProjectSecretsKeys(self, connection_name, project_name):
|
||||
"""Return the private and public secrets keys for the project
|
||||
|
||||
A new key will be created if necessary.
|
||||
|
||||
:returns: A tuple (private_key, public_key)
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class FileKeyStorage(KeyStorage):
|
||||
log = logging.getLogger("zuul.FileKeyStorage")
|
||||
current_version = MigrationV1
|
||||
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
migration = self.current_version()
|
||||
migration.verifyAndUpgrade(root)
|
||||
|
||||
def getProjectSecretsKeyFile(self, connection, project, version=None):
|
||||
"""Return the path to the private key used for the project's secrets"""
|
||||
# We don't actually support multiple versions yet
|
||||
if version is None:
|
||||
version = '0'
|
||||
return os.path.join(self.root, 'secrets', 'project',
|
||||
connection, project, version + '.pem')
|
||||
|
||||
def getProjectSSHKeyFile(self, connection, project, version=None):
|
||||
"""Return the path to the private ssh key for the project"""
|
||||
# We don't actually support multiple versions yet
|
||||
if version is None:
|
||||
version = '0'
|
||||
return os.path.join(self.root, 'ssh', 'project',
|
||||
connection, project, version + '.pem')
|
||||
|
||||
def hasProjectSSHKeys(self, connection_name, project_name):
|
||||
return os.path.isfile(
|
||||
self.getProjectSSHKeyFile(connection_name, project_name))
|
||||
|
||||
def setProjectSSHKey(self, connection_name, project_name, private_key):
|
||||
private_key_file = self.getProjectSSHKeyFile(connection_name,
|
||||
project_name)
|
||||
key_dir = os.path.dirname(private_key_file)
|
||||
if not os.path.isdir(key_dir):
|
||||
os.makedirs(key_dir, 0o700)
|
||||
|
||||
private_key.write_private_key_file(private_key_file)
|
||||
return private_key_file
|
||||
|
||||
def getProjectSSHKeys(self, connection_name, project_name):
|
||||
private_key_file = self._ensureSSHKeyFile(connection_name,
|
||||
project_name)
|
||||
|
||||
key = paramiko.RSAKey.from_private_key_file(private_key_file)
|
||||
with open(private_key_file, 'r') as f:
|
||||
private_key = f.read()
|
||||
public_key = key.get_base64()
|
||||
|
||||
return (private_key, 'ssh-rsa ' + public_key)
|
||||
|
||||
def _ensureSSHKeyFile(self, connection_name, project_name):
|
||||
private_key_file = self.getProjectSSHKeyFile(connection_name,
|
||||
project_name)
|
||||
if os.path.exists(private_key_file):
|
||||
return private_key_file
|
||||
|
||||
self.log.info("Generating SSH public key for project %s", project_name)
|
||||
pk = paramiko.RSAKey.generate(bits=RSA_KEY_SIZE)
|
||||
return self.setProjectSSHKey(connection_name, project_name, pk)
|
||||
|
||||
def hasProjectSecretsKeys(self, connection_name, project_name):
|
||||
return os.path.isfile(
|
||||
self.getProjectSecretsKeyFile(connection_name, project_name))
|
||||
|
||||
def setProjectSecretsKey(self, connection_name, project_name, private_key):
|
||||
filename = self.getProjectSecretsKeyFile(connection_name, project_name)
|
||||
key_dir = os.path.dirname(filename)
|
||||
if not os.path.isdir(key_dir):
|
||||
os.makedirs(key_dir, 0o700)
|
||||
|
||||
# Dump keys to filesystem. We only save the private key
|
||||
# because the public key can be constructed from it.
|
||||
self.log.info("Saving RSA keypair for project %s to %s",
|
||||
project_name, filename)
|
||||
|
||||
pem_private_key = encryption.serialize_rsa_private_key(private_key)
|
||||
# Ensure private key is read/write for zuul user only.
|
||||
with open(os.open(filename,
|
||||
os.O_CREAT | os.O_WRONLY, 0o600), 'wb') as f:
|
||||
f.write(pem_private_key)
|
||||
return filename
|
||||
|
||||
def getProjectSecretsKeys(self, connection_name, project_name):
|
||||
private_key_file = self._ensureSecretsKeyFile(connection_name,
|
||||
project_name)
|
||||
|
||||
# Load keypair
|
||||
with open(private_key_file, "rb") as f:
|
||||
return encryption.deserialize_rsa_keypair(f.read())
|
||||
|
||||
def _ensureSecretsKeyFile(self, connection_name, project_name):
|
||||
filename = self.getProjectSecretsKeyFile(connection_name, project_name)
|
||||
if os.path.isfile(filename):
|
||||
return filename
|
||||
|
||||
self.log.info("Generating RSA keypair for project %s", project_name)
|
||||
private_key, public_key = encryption.generate_rsa_keypair()
|
||||
|
||||
return self.setProjectSecretsKey(connection_name, project_name,
|
||||
private_key)
|
||||
|
||||
|
||||
class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
|
||||
log = logging.getLogger("zuul.ZooKeeperKeyStorage")
|
||||
SECRETS_PATH = "/keystorage/{}/{}/secrets"
|
||||
SSH_PATH = "/keystorage/{}/{}/ssh"
|
||||
|
||||
|
|
@ -267,31 +36,58 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
|
|||
super().__init__(zookeeper_client)
|
||||
self.password = password
|
||||
self.password_bytes = password.encode("utf-8")
|
||||
self.backup = backup
|
||||
|
||||
def _walk(self, root):
|
||||
ret = []
|
||||
children = self.kazoo_client.get_children(root)
|
||||
if children:
|
||||
for child in children:
|
||||
path = '/'.join([root, child])
|
||||
ret.extend(self._walk(path))
|
||||
else:
|
||||
data, _ = self.kazoo_client.get(root)
|
||||
try:
|
||||
json.loads(data)
|
||||
ret.append((root, data))
|
||||
except Exception:
|
||||
self.log.error("Unable to load keys at %s", root)
|
||||
# Keep processing exports
|
||||
return ret
|
||||
|
||||
def exportKeys(self):
|
||||
paths = self._walk('/keystorage')
|
||||
# Drop /keystorage from the start of the path
|
||||
return [(path.split('/', 2)[2], data) for (path, data) in paths]
|
||||
|
||||
def importKeys(self, import_list):
|
||||
for path, data in import_list:
|
||||
try:
|
||||
json.loads(data)
|
||||
except Exception:
|
||||
self.log.error("Unable to load keys at %s", path)
|
||||
# Abort import on bad data
|
||||
raise
|
||||
path = '/'.join(['/keystorage', path])
|
||||
try:
|
||||
self.kazoo_client.create(path, value=data, makepath=True)
|
||||
except kazoo.exceptions.NodeExistsError:
|
||||
self.kazoo_client.set(path, value=data)
|
||||
|
||||
def getSSHKeysPath(self, connection_name, project_name):
|
||||
key_project_name = strings.unique_project_name(project_name)
|
||||
key_path = self.SSH_PATH.format(connection_name, key_project_name)
|
||||
return key_path
|
||||
|
||||
@cachetools.cached(cache={})
|
||||
def getProjectSSHKeys(self, connection_name, project_name):
|
||||
key_project_name = strings.unique_project_name(project_name)
|
||||
key_path = self.SSH_PATH.format(connection_name, key_project_name)
|
||||
key_path = self.getSSHKeysPath(connection_name, project_name)
|
||||
|
||||
try:
|
||||
key = self._getSSHKey(key_path)
|
||||
except kazoo.exceptions.NoNodeError:
|
||||
self.log.debug("Could not find existing SSH key for %s/%s",
|
||||
connection_name, project_name)
|
||||
if self.backup and self.backup.hasProjectSSHKeys(
|
||||
connection_name, project_name
|
||||
):
|
||||
self.log.info("Using SSH key for %s/%s from backup key store",
|
||||
connection_name, project_name)
|
||||
pk, _ = self.backup.getProjectSSHKeys(connection_name,
|
||||
project_name)
|
||||
with io.StringIO(pk) as o:
|
||||
key = paramiko.RSAKey.from_private_key(o)
|
||||
else:
|
||||
self.log.info("Generating a new SSH key for %s/%s",
|
||||
connection_name, project_name)
|
||||
key = paramiko.RSAKey.generate(bits=RSA_KEY_SIZE)
|
||||
self.log.info("Generating a new SSH key for %s/%s",
|
||||
connection_name, project_name)
|
||||
key = paramiko.RSAKey.generate(bits=RSA_KEY_SIZE)
|
||||
key_version = 0
|
||||
key_created = int(time.time())
|
||||
|
||||
|
|
@ -302,10 +98,6 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
|
|||
# creating the same SSH key.
|
||||
key = self._getSSHKey(key_path)
|
||||
|
||||
# Make sure the SSH key is also stored in the backup keystore
|
||||
if self.backup:
|
||||
self.backup.setProjectSSHKey(connection_name, project_name, key)
|
||||
|
||||
with io.StringIO() as o:
|
||||
key.write_private_key(o)
|
||||
private_key = o.getvalue()
|
||||
|
|
@ -337,32 +129,27 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
|
|||
data = json.dumps(keydata).encode("utf-8")
|
||||
self.kazoo_client.create(key_path, value=data, makepath=True)
|
||||
|
||||
@cachetools.cached(cache={})
|
||||
def getProjectSecretsKeys(self, connection_name, project_name):
|
||||
def getProjectSecretsKeysPath(self, connection_name, project_name):
|
||||
key_project_name = strings.unique_project_name(project_name)
|
||||
key_path = self.SECRETS_PATH.format(connection_name, key_project_name)
|
||||
return key_path
|
||||
|
||||
@cachetools.cached(cache={})
|
||||
def getProjectSecretsKeys(self, connection_name, project_name):
|
||||
key_path = self.getProjectSecretsKeysPath(
|
||||
connection_name, project_name)
|
||||
|
||||
try:
|
||||
pem_private_key = self._getSecretsKeys(key_path)
|
||||
except kazoo.exceptions.NoNodeError:
|
||||
self.log.debug("Could not find existing secrets key for %s/%s",
|
||||
connection_name, project_name)
|
||||
if self.backup and self.backup.hasProjectSecretsKeys(
|
||||
connection_name, project_name):
|
||||
self.log.info(
|
||||
"Using secrets key for %s/%s from backup key store",
|
||||
connection_name, project_name)
|
||||
private_key, public_key = self.backup.getProjectSecretsKeys(
|
||||
connection_name, project_name)
|
||||
else:
|
||||
self.log.info("Generating a new secrets key for %s/%s",
|
||||
connection_name, project_name)
|
||||
private_key, public_key = encryption.generate_rsa_keypair()
|
||||
|
||||
self.log.info("Generating a new secrets key for %s/%s",
|
||||
connection_name, project_name)
|
||||
private_key, public_key = encryption.generate_rsa_keypair()
|
||||
pem_private_key = encryption.serialize_rsa_private_key(
|
||||
private_key, self.password_bytes)
|
||||
key_version = 0
|
||||
key_created = int(time.time())
|
||||
|
||||
try:
|
||||
self._storeSecretsKeys(key_path, pem_private_key,
|
||||
key_version, key_created)
|
||||
|
|
@ -374,11 +161,6 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
|
|||
private_key, public_key = encryption.deserialize_rsa_keypair(
|
||||
pem_private_key, self.password_bytes)
|
||||
|
||||
# Make sure the private key is also stored in the backup keystore
|
||||
if self.backup:
|
||||
self.backup.setProjectSecretsKey(connection_name, project_name,
|
||||
private_key)
|
||||
|
||||
return private_key, public_key
|
||||
|
||||
def _getSecretsKeys(self, key_path):
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ from zuul.lib import commandsocket
|
|||
from zuul.lib.ansible import AnsibleManager
|
||||
from zuul.lib.config import get_default
|
||||
from zuul.lib.gear_utils import getGearmanFunctions
|
||||
from zuul.lib.keystorage import FileKeyStorage, ZooKeeperKeyStorage
|
||||
from zuul.lib.keystorage import KeyStorage
|
||||
from zuul.lib.logutil import get_annotated_logger
|
||||
from zuul.lib.queue import NamedQueue
|
||||
from zuul.lib.statsd import get_statsd, normalize_statsd_name
|
||||
|
|
@ -230,10 +230,9 @@ class Scheduler(threading.Thread):
|
|||
|
||||
def start(self):
|
||||
super(Scheduler, self).start()
|
||||
self.keystore = ZooKeeperKeyStorage(
|
||||
self.keystore = KeyStorage(
|
||||
self.zk_client,
|
||||
password=self._get_key_store_password(),
|
||||
backup=FileKeyStorage(self._get_key_dir()))
|
||||
password=self._get_key_store_password())
|
||||
|
||||
self._command_running = True
|
||||
self.log.debug("Starting command processor")
|
||||
|
|
@ -790,19 +789,6 @@ class Scheduler(threading.Thread):
|
|||
except KeyError:
|
||||
raise RuntimeError("No key store password configured!")
|
||||
|
||||
def _get_key_dir(self):
|
||||
state_dir = get_default(self.config, 'scheduler', 'state_dir',
|
||||
'/var/lib/zuul', expand_user=True)
|
||||
key_dir = os.path.join(state_dir, 'keys')
|
||||
if not os.path.exists(key_dir):
|
||||
os.mkdir(key_dir, 0o700)
|
||||
st = os.stat(key_dir)
|
||||
mode = st.st_mode & 0o777
|
||||
if mode != 0o700:
|
||||
raise Exception("Project key directory %s must be mode 0700; "
|
||||
"current mode is %o" % (key_dir, mode))
|
||||
return key_dir
|
||||
|
||||
def _checkTenantSourceConf(self, config):
|
||||
tenant_config = None
|
||||
script = False
|
||||
|
|
@ -2014,3 +2000,42 @@ class Scheduler(threading.Thread):
|
|||
# Release the semaphore in any case
|
||||
tenant = buildset.item.pipeline.tenant
|
||||
tenant.semaphore_handler.release(item, job)
|
||||
|
||||
|
||||
def export_keys(config, path):
|
||||
zk_client = ZooKeeperClient.fromConfig(config)
|
||||
zk_client.connect()
|
||||
try:
|
||||
password = config["keystore"]["password"]
|
||||
except KeyError:
|
||||
raise RuntimeError("No key store password configured!")
|
||||
keystore = KeyStorage(zk_client, password=password)
|
||||
for key_path, key_data in keystore.exportKeys():
|
||||
print(key_path)
|
||||
key_root, fn = key_path.rsplit('/', 1)
|
||||
root = os.path.join(path, key_root)
|
||||
os.makedirs(root, exist_ok=True)
|
||||
with open(os.path.join(root, fn), 'wb') as f:
|
||||
f.write(key_data)
|
||||
|
||||
|
||||
def import_keys(config, path):
|
||||
zk_client = ZooKeeperClient.fromConfig(config)
|
||||
zk_client.connect()
|
||||
try:
|
||||
password = config["keystore"]["password"]
|
||||
except KeyError:
|
||||
raise RuntimeError("No key store password configured!")
|
||||
keystore = KeyStorage(zk_client, password=password)
|
||||
import_list = []
|
||||
for (dirpath, dirnames, filenames) in os.walk(path):
|
||||
for fn in filenames:
|
||||
key_root = dirpath[len(path):]
|
||||
if key_root.startswith('/'):
|
||||
key_root = key_root[1:]
|
||||
key_root = os.path.join(key_root, fn)
|
||||
print(key_root)
|
||||
with open(os.path.join(dirpath, fn), 'rb') as f:
|
||||
key_data = f.read()
|
||||
import_list.append((key_root, key_data))
|
||||
keystore.importKeys(import_list)
|
||||
|
|
|
|||
Loading…
Reference in New Issue