zuul/zuul
Code Issues Proposed changes

Merge "Support keystore password rotation"

Browse Source
This commit is contained in:
Zuul authored and Gerrit Code Review committed 2026-10-01 21:56:27 +00:00
2 parents 3f48320e1d 4b6f3e7700
commit 4e9283a465
15 files changed
+426 -218
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

doc/source/configuration.rst
+31
View File
@@ -279,6 +279,37 @@ The following sections of ``zuul.conf`` are used by the scheduler:
Encryption password for private data stored in Zookeeper.
Typically this is a single value. To rotate keys, multiple
values may be supplied, each on its own line. The first value
is always the desired (new) key, subsequent values are older
keys. Once all of the components in the cluster have been
restarted with the new values, all existing key material will be
re-encrypted with the new key. After that point, old keys may
be removed.
For example, with an original configuration of:
.. code-block:: ini
[keystore]
password = password1
Zuul may be instructed to rotate to `password` with:
.. code-block:: ini
[keystore]
password =
password2
password1
After rotation, this may be simplified to:
.. code-block:: ini
[keystore]
password = password2
.. attr:: scheduler
.. attr:: command_socket
tests/base.py
+47 -28
View File
@@ -71,7 +71,7 @@ from kazoo.exceptions import NoNodeError
from zuul import model
from zuul.model import (
BuildRequest, MergeRequest, WebInfo, HoldRequest
BuildRequest, MergeRequest, WebInfo, HoldRequest, KeystorePasswordManager
)
from zuul.driver.zuul import ZuulDriver
@@ -2617,29 +2617,9 @@ class ZuulTestCase(BaseTestCase):
self._configureMqtt()
self._configureElasticsearch()
executor_connections = TestConnectionRegistry(
self.config, self.test_config,
self.additional_event_queues,
self.upstream_root, self.poller_events,
self.git_url_with_auth, self.addCleanup)
executor_connections.configure(self.config, sources=True)
self.executor_api = TestingExecutorApi(self.zk_client)
self.merger_api = TestingMergerApi(self.zk_client)
self.executor_server = RecordingExecutorServer(
self.config,
executor_connections,
jobdir_root=self.jobdir_root,
_run_ansible=self.run_ansible,
_test_root=self.test_root,
keep_jobdir=KEEP_TEMPDIRS,
log_console_port=self.log_console_port)
for return_data in self.test_config.return_data:
self.executor_server.returnData(
return_data['job'],
return_data['ref'],
return_data['data'],
)
self.executor_server.start()
self.executor_server = self.createExecutor(self.config)
self.history = self.executor_server.build_history
self.builds = self.executor_server.running_builds
@@ -2698,6 +2678,40 @@ class ZuulTestCase(BaseTestCase):
launcher.start()
return launcher
def createExecutor(self, config=None):
if config is None:
config = self.config
executor_config = copyConfig(config)
executor_connections = TestConnectionRegistry(
self.config, self.test_config,
self.additional_event_queues,
self.upstream_root, self.poller_events,
self.git_url_with_auth, self.addCleanup)
executor_connections.configure(executor_config, sources=True)
executor = RecordingExecutorServer(
executor_config,
connections=executor_connections,
jobdir_root=self.jobdir_root,
_run_ansible=self.run_ansible,
_test_root=self.test_root,
log_console_port=self.log_console_port)
for return_data in self.test_config.return_data:
executor.returnData(
return_data['job'],
return_data['ref'],
return_data['data'],
)
executor.start()
def shutdown_executor():
if executor._running:
executor.hold_jobs_in_build = False
executor.release()
executor.stop()
executor.join()
self.addCleanup(shutdown_executor)
return executor
def createZKContext(self, lock=None):
if lock is None:
# Just make sure the lock is acquired
@@ -2920,9 +2934,11 @@ class ZuulTestCase(BaseTestCase):
if self.create_project_keys:
return
password = self.config.get("keystore", "password")
keystore_password_manager = KeystorePasswordManager(self.config)
keystore_password_manager.setPassword(
keystore_password_manager.password_hashes)
keystore = zuul.lib.keystorage.KeyStorage(
self.zk_client, password=password, start_cache=False)
self.zk_client, keystore_password_manager, start_cache=False)
# This is only needed for the keystore1 -> keystore3 migration
COMPONENT_REGISTRY.create(self.zk_client)
@@ -2965,9 +2981,11 @@ class ZuulTestCase(BaseTestCase):
def setupTenantKeys(self, tenant_name):
# Make sure we set up an RSA key for the tenant so that we
# don't spend time generating one:
password = self.config.get("keystore", "password")
keystore_password_manager = KeystorePasswordManager(self.config)
keystore_password_manager.setPassword(
keystore_password_manager.password_hashes)
keystore = zuul.lib.keystorage.KeyStorage(
self.zk_client, password=password, start_cache=False)
self.zk_client, keystore_password_manager, start_cache=False)
import_keys = {}
import_data = {'keys': import_keys}
@@ -3137,8 +3155,9 @@ class ZuulTestCase(BaseTestCase):
self.merge_server.stop()
self.merge_server.join()
self.executor_server.stop()
self.executor_server.join()
if self.executor_server._running:
self.executor_server.stop()
self.executor_server.join()
self.launcher.stop()
self.launcher.join()
if self.validate_tenants is None:
tests/unit/test_configloader.py
+10 -3
View File
@@ -27,7 +27,12 @@ from zuul.configloader import (
ConfigLoader, safe_load_yaml
)
from zuul.exceptions import AuthZRoleNotFoundError, AuthZRuleNotFoundError
from zuul.model import Abide, MergeRequest, SourceContext
from zuul.model import (
Abide,
KeystorePasswordManager,
MergeRequest,
SourceContext,
)
from zuul.zk.components import COMPONENT_REGISTRY
from zuul.zk.locks import tenant_read_lock
@@ -121,9 +126,11 @@ class TenantParserTestCase(ZuulTestCase):
CONFIG_SET = UNTRUSTED_SET | PROVIDER_SET
def setupAllProjectKeys(self, config: ConfigParser):
password = self.config.get("keystore", "password")
kpm = KeystorePasswordManager(self.config)
kpm.setPassword(set(kpm.password_hashes))
keystore = zuul.lib.keystorage.KeyStorage(
self.zk_client, password=password, start_cache=False)
self.zk_client, kpm, start_cache=False)
# This is only needed for the keystore1 -> keystore3 migration
COMPONENT_REGISTRY.create(self.zk_client)
tests/unit/test_keystorage.py
+77 -9
View File
@@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from configparser import ConfigParser
import io
import time
@@ -21,6 +22,7 @@ from cryptography.hazmat.primitives.asymmetric.rsa import (
RSAPrivateKey,
)
from zuul.model import KeystorePasswordManager
from zuul.lib import encryption
from zuul.lib import keystorage3
from zuul.zk import ZooKeeperClient
@@ -30,7 +32,9 @@ from zuul.zk.components import (
)
from tests.base import (
copyConfig,
BaseTestCase,
ZuulTestCase,
ZOOKEEPER_SESSION_TIMEOUT,
)
@@ -56,9 +60,15 @@ class TestKeyStorage3(BaseTestCase):
COMPONENT_REGISTRY.create(self.zk_client)
self.password = 'keystorage_password'
self.config = ConfigParser()
self.config.add_section('keystore')
self.config['keystore']['password'] = self.password
self.kpm = KeystorePasswordManager(self.config)
self.kpm.setPassword(set(self.kpm.password_hashes))
def test_project_ssh_keys(self):
start_time = int(time.time())
ks = keystorage3.KeyStorage3(self.zk_client, self.password)
ks = keystorage3.KeyStorage3(self.zk_client, self.kpm)
self.addCleanup(ks.stop)
# This will generate a new ssh key
info = ks.getProjectSSHKeyInfo('gerrit', 'org/project')
@@ -83,7 +93,7 @@ class TestKeyStorage3(BaseTestCase):
self.assertTrue(info is info2)
# Another client to read it from ZK
ks2 = keystorage3.KeyStorage3(self.zk_client, self.password)
ks2 = keystorage3.KeyStorage3(self.zk_client, self.kpm)
self.addCleanup(ks2.stop)
ks2_info = ks2.getProjectSSHKeyInfo('gerrit', 'org/project')
@@ -98,7 +108,7 @@ class TestKeyStorage3(BaseTestCase):
def test_tenant_ssh_keys(self):
start_time = int(time.time())
ks = keystorage3.KeyStorage3(self.zk_client, self.password)
ks = keystorage3.KeyStorage3(self.zk_client, self.kpm)
self.addCleanup(ks.stop)
# This will generate a new ssh key
info = ks.getTenantSSHKeyInfo('tenant1')
@@ -123,7 +133,7 @@ class TestKeyStorage3(BaseTestCase):
self.assertTrue(info is info2)
# Another client to read it from ZK
ks2 = keystorage3.KeyStorage3(self.zk_client, self.password)
ks2 = keystorage3.KeyStorage3(self.zk_client, self.kpm)
self.addCleanup(ks2.stop)
ks2_info = ks2.getTenantSSHKeyInfo('tenant1')
@@ -138,7 +148,7 @@ class TestKeyStorage3(BaseTestCase):
def test_project_secret_keys(self):
start_time = int(time.time())
ks = keystorage3.KeyStorage3(self.zk_client, self.password)
ks = keystorage3.KeyStorage3(self.zk_client, self.kpm)
self.addCleanup(ks.stop)
# This will generate a new secrets key
info = ks.getProjectSecretsKeyInfo('gerrit', 'org/project')
@@ -163,7 +173,7 @@ class TestKeyStorage3(BaseTestCase):
self.assertTrue(info is info2)
# Another client to read it from ZK
ks2 = keystorage3.KeyStorage3(self.zk_client, self.password)
ks2 = keystorage3.KeyStorage3(self.zk_client, self.kpm)
self.addCleanup(ks2.stop)
ks2_info = ks2.getProjectSecretsKeyInfo('gerrit', 'org/project')
@@ -179,7 +189,7 @@ class TestKeyStorage3(BaseTestCase):
ks2_key1.private_key.private_numbers())
def test_rotate_project_ssh_keys(self):
ks = keystorage3.KeyStorage3(self.zk_client, self.password)
ks = keystorage3.KeyStorage3(self.zk_client, self.kpm)
self.addCleanup(ks.stop)
# This will generate a new secrets key
info = ks.getProjectSSHKeyInfo('gerrit', 'org/project')
@@ -207,7 +217,7 @@ class TestKeyStorage3(BaseTestCase):
self.assertEqual(1, key3.version)
def test_rotate_tenant_ssh_keys(self):
ks = keystorage3.KeyStorage3(self.zk_client, self.password)
ks = keystorage3.KeyStorage3(self.zk_client, self.kpm)
self.addCleanup(ks.stop)
# This will generate a new secrets key
info = ks.getTenantSSHKeyInfo('tenant-one')
@@ -235,7 +245,7 @@ class TestKeyStorage3(BaseTestCase):
self.assertEqual(1, key3.version)
def test_rotate_project_secret_keys(self):
ks = keystorage3.KeyStorage3(self.zk_client, self.password)
ks = keystorage3.KeyStorage3(self.zk_client, self.kpm)
self.addCleanup(ks.stop)
# This will generate a new secrets key
info = ks.getProjectSecretsKeyInfo('gerrit', 'org/project')
@@ -261,3 +271,61 @@ class TestKeyStorage3(BaseTestCase):
self.assertEqual(1, len(info.keys))
key3 = info.keys[0]
self.assertEqual(1, key3.version)
class TestKeystorePasswordRotation(ZuulTestCase):
tenant_config_file = 'config/one-project/main.yaml'
def test_keystore_password_rotation(self):
first = self.scheds.first
ks = first.sched.keystore
info = ks.getProjectSecretsKeyInfo('gerrit', 'org/project')
key = info.keys[0]
encryption.deserialize_rsa_keypair(
key._encrypted_private_key.encode('utf8'), b'keystorepassword')
ssh_info = ks.getProjectSSHKeyInfo('gerrit', 'org/project')
ssh_key = ssh_info.keys[0]
with io.StringIO(ssh_key._encrypted_private_key) as o:
paramiko.RSAKey.from_private_key(o, 'keystorepassword')
new_pw = "newpassword\nkeystorepassword\n"
self.config['keystore']['password'] = new_pw
# Replace scheduler
first.sched.stop()
first.sched.join()
self.createScheduler()
self.scheds.instances.pop(0)
first = self.scheds.first
first.start()
self.waitUntilSettled()
# Replace Executor
self.executor_server.stop()
self.executor_server.join()
self.executor_server = self.createExecutor()
# Verify that the item has been re-keyed
ks = first.sched.keystore
info = ks.getProjectSecretsKeyInfo('gerrit', 'org/project')
key = info.keys[0]
encryption.deserialize_rsa_keypair(
key._encrypted_private_key.encode('utf8'), b'newpassword')
# Also verify we wrote the update to zk
config = copyConfig(self.config)
config['keystore']['password'] = "newpassword"
kpm = KeystorePasswordManager(config)
kpm.setPassword(set(kpm.password_hashes))
ks = keystorage3.KeyStorage3(self.zk_client, kpm)
extra = dict(keystore_password_manager=kpm)
ctx = ks.createZKContext()
data, zstat = self.zk_client.client.get(info._path)
keystorage3.ZuulSecretKeyInfo._fromRaw(ctx, data, zstat, extra=extra)
data, zstat = self.zk_client.client.get(ssh_info._path)
keystorage3.ZuulSSHKeyInfo._fromRaw(ctx, data, zstat, extra=extra)
self.waitUntilSettled()
tests/unit/test_scheduler.py
+1 -21
View File
@@ -44,7 +44,6 @@ from tests.base import (
FIXTURE_DIR,
RecordingExecutorServer,
SSLZuulTestCase,
TestConnectionRegistry,
ZuulTestCase,
iterate_timeout,
okay_tracebacks,
@@ -97,26 +96,7 @@ class TestSchedulerZone(ZuulTestCase):
config.remove_option('executor', 'zone')
config.set('executor', 'command_socket',
os.path.join(self.test_root, 'executor2.socket'))
executor_connections = TestConnectionRegistry(
self.config, self.test_config, self.additional_event_queues,
self.upstream_root, self.poller_events,
self.git_url_with_auth, self.addCleanup)
executor_connections.configure(self.config, sources=True)
self.executor_server_unzoned = RecordingExecutorServer(
config,
connections=executor_connections,
jobdir_root=self.jobdir_root,
_run_ansible=self.run_ansible,
_test_root=self.test_root,
log_console_port=self.log_console_port)
self.executor_server_unzoned.start()
self.addCleanup(self._shutdown_executor)
def _shutdown_executor(self):
self.executor_server_unzoned.hold_jobs_in_build = False
self.executor_server_unzoned.release()
self.executor_server_unzoned.stop()
self.executor_server_unzoned.join()
self.executor_server_unzoned = self.createExecutor(config)
def setup_config(self, config_file: str):
config = super(TestSchedulerZone, self).setup_config(config_file)
tests/unit/test_v3.py
+1 -1
View File
@@ -4389,7 +4389,7 @@ class FunctionalAnsibleMixIn(object):
# Add a site variables file, used by check-vars
path = os.path.join(FIXTURE_DIR, 'config', 'ansible',
'variables.yaml')
self.config.set('executor', 'variables', path)
self.executor_server.config.set('executor', 'variables', path)
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
tests/upgrade/test_upgrade_old.py
+3 -2
View File
@@ -45,8 +45,9 @@ class TestUpgradeOld(LauncherBaseTestCase):
self.merge_server.stop()
self.merge_server.join()
self.executor_server.stop()
self.executor_server.join()
if self.executor_server._running:
self.executor_server.stop()
self.executor_server.join()
self.launcher.stop()
self.launcher.join()
self.statsd.stop()
zuul/cmd/client.py
+24 -71
View File
@@ -38,6 +38,7 @@ from zuul.model import (
Abide,
ImageBuildArtifact,
ImageUpload,
KeystorePasswordManager,
Pipeline,
PipelineChangeList,
PipelineState,
@@ -1202,16 +1203,21 @@ class Client(zuul.cmd.ZuulApp):
finally:
sys.exit(err_code)
def getKeystore(self, zk_client):
try:
self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
kpm = KeystorePasswordManager(self.config)
kpm.setPassword(set(kpm.password_hashes))
return KeyStorage(zk_client, kpm)
def export_keys(self):
logging.basicConfig(level=logging.INFO)
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
export = keystore.exportKeys()
with open(os.open(self.args.path,
os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
@@ -1223,11 +1229,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
with open(self.args.path, 'r') as f:
import_data = json.load(f)
keystore.importKeys(import_data, self.args.force)
@@ -1241,11 +1243,7 @@ class Client(zuul.cmd.ZuulApp):
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
keystore.copyProjectSSHKeys(args.src_connection,
args.src_project,
@@ -1267,12 +1265,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client.connect()
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
keystore.deleteProjectSSHKeys(args.connection, args.project)
keystore.deleteProjectSecretsKeys(args.connection, args.project)
@@ -1286,11 +1279,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
keystore.addProjectSSHKey(args.connection, args.project)
sys.exit(0)
@@ -1300,11 +1289,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
keystore.addProjectSecretsKey(args.connection, args.project)
sys.exit(0)
@@ -1314,11 +1299,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
keystore.addTenantSSHKey(args.tenant)
sys.exit(0)
@@ -1328,11 +1309,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
info = keystore.getProjectSSHKeyInfo(args.connection, args.project,
create=False)
@@ -1347,11 +1324,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
info = keystore.getProjectSecretsKeyInfo(args.connection, args.project,
create=False)
@@ -1366,11 +1339,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
info = keystore.getTenantSSHKeyInfo(args.tenant, create=False)
table = prettytable.PrettyTable(field_names=['Version', 'Created'])
@@ -1384,11 +1353,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
keystore.deleteProjectSSHKey(
args.connection, args.project, args.version)
@@ -1399,11 +1364,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
keystore.deleteProjectSecretsKey(
args.connection, args.project, args.version)
@@ -1414,11 +1375,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
COMPONENT_REGISTRY.create(zk_client)
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
args = self.args
keystore.deleteTenantSSHKey(args.tenant, args.version)
sys.exit(0)
@@ -1428,11 +1385,7 @@ class Client(zuul.cmd.ZuulApp):
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
try:
password = self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
keystore = KeyStorage(zk_client, password=password)
keystore = self.getKeystore(zk_client)
algorithm = self.args.algorithm
keystore.deleteOidcSigningKeys(algorithm)
self.log.info("Delete OIDC signing keys for %s", algorithm)
zuul/executor/server.py
+15 -5
View File
@@ -93,6 +93,7 @@ from zuul.model import (
BuildStatusEvent,
FrozenJob,
Job,
KeystorePasswordManager,
RepoState,
)
import zuul.model
@@ -4129,19 +4130,28 @@ class ExecutorServer(BaseMergeServer):
self.keep_jobdir = keep_jobdir
self.jobdir_root = jobdir_root
self.keystore = KeyStorage(
self.zk_client,
password=self._get_key_store_password())
self._running = False
self._command_running = False
# TODOv3(mordred): make the executor name more unique --
# perhaps hostname+pid.
self.hostname = get_default(self.config, 'executor', 'hostname',
socket.getfqdn())
self.keystore_password_manager = KeystorePasswordManager(self.config)
self.component_info = ExecutorComponent(
self.zk_client, self.hostname, version=get_version_string())
self.zk_client, self.hostname, version=get_version_string(),
keystore_passwords=self.keystore_password_manager.password_hashes,
)
self.component_info.register()
COMPONENT_REGISTRY.create(self.zk_client)
component_registry = COMPONENT_REGISTRY.create(self.zk_client)
# This is mostly for unit tests, where we can have an empty,
# initialized registry object.
component_registry.waitForComponent(self.component_info)
self.keystore_password_manager.connectToRegistry(component_registry)
self.keystore = KeyStorage(
self.zk_client,
self.keystore_password_manager)
self.secrets_cache = cachetools.LFUCache(maxsize=128)
self.zk_context = ZKContext(self.zk_client, None, None, self.log)
self.monitoring_server = MonitoringServer(self.config, 'executor',
zuul/lib/keystorage.py
+25 -9
View File
@@ -31,17 +31,27 @@ RSA_KEY_SIZE = 2048
class KeyStorage:
log = logging.getLogger("zuul.KeyStorage")
def __init__(self, zookeeper_client, password, start_cache=True):
self.password = password
self.password_bytes = password.encode('utf8')
def __init__(self, zookeeper_client, keystore_password_manager,
start_cache=True):
self.kp_manager = keystore_password_manager
self.all_passwords = self.kp_manager.passwords
self.all_passwords_bytes = self.kp_manager.passwords_bytes
self.client = zookeeper_client
self.keystorage1 = KeyStorage1(zookeeper_client, password)
self.keystorage3 = KeyStorage3(zookeeper_client, password)
self.keystorage1 = KeyStorage1(zookeeper_client, self.password)
self.keystorage3 = KeyStorage3(zookeeper_client, self.kp_manager)
if start_cache:
self.oidc_signing_keys_cache = OIDCSigningKeysCache(self.client)
else:
self.oidc_signing_keys_cache = None
@property
def password(self):
return self.kp_manager.password
@property
def password_bytes(self):
return self.kp_manager.password_bytes
@property
def _keystorage(self):
# TODO: When we remove this backwards compat code, we
@@ -265,10 +275,16 @@ class KeyStorage:
pem_private_key = latest_key["private_key"].encode("utf-8")
version = latest_key["version"]
private_key, public_key = encryption.deserialize_rsa_keypair(
pem_private_key, self.password_bytes)
return private_key, public_key, version
# TODO use old keys:
for password_bytes in self.all_passwords_bytes:
try:
private_key, public_key =\
encryption.deserialize_rsa_keypair(
pem_private_key, password_bytes)
return private_key, public_key, version
except ValueError:
pass
raise ValueError("Decryption failed")
def deleteOidcSigningKeys(self, algorithm):
"""Delete the complete internal data structure"""
zuul/lib/keystorage3.py
+88 -51
View File
@@ -38,6 +38,11 @@ class ZuulSSHKey:
self.created = None
self.version = None
def _rekey(self, password):
with io.StringIO() as o:
self._key_object.write_private_key(o, password)
self._encrypted_private_key = o.getvalue()
@classmethod
def new(cls, password, version):
self = cls()
@@ -85,6 +90,10 @@ class ZuulSecretKey:
self.created = None
self.version = None
def _rekey(self, password):
self._encrypted_private_key = encryption.serialize_rsa_private_key(
self.private_key, password).decode("utf-8")
@classmethod
def new(cls, password, version):
self = cls()
@@ -142,75 +151,94 @@ class ZuulKeyInfo(ZKObject):
def private_keys(self):
return [k.private_key for k in self.keys]
class ZuulSSHKeyInfo(ZuulKeyInfo):
def deserialize(self, data, context, extra=None):
data = super().deserialize(data, context, extra)
data['keys'] = [ZuulSSHKey.fromDict(k, extra['password'])
for k in data['keys']]
self._handleKeys(data, extra)
return data
def updateFromDict(self, context, extra=None, **data):
if 'keys' in data:
data['keys'] = [ZuulSSHKey.fromDict(k, extra['password'])
for k in data['keys']]
self._handleKeys(data, extra)
self.updateAttributes(context, **data)
@classmethod
def newFromDict(klass, context, extra=None, **data):
if 'keys' in data:
data['keys'] = [ZuulSSHKey.fromDict(k, extra['password'])
for k in data['keys']]
def newFromDict(cls, context, extra=None, **data):
cls._handleKeys(data, extra)
return super().new(context, **data)
class ZuulSSHKeyInfo(ZuulKeyInfo):
@classmethod
def _handleKeys(cls, data, extra):
kp_manager = extra['keystore_password_manager']
keys = []
for keydata in data.get('keys', []):
key = None
error = None
for password in kp_manager.passwords:
try:
key = ZuulSSHKey.fromDict(keydata, password)
break
except paramiko.ssh_exception.SSHException as e:
error = e
if key is None:
raise error
if password != kp_manager.password:
data['_rekey'] = True
key._rekey(kp_manager.password)
keys.append(key)
data['keys'] = keys
class ZuulSecretKeyInfo(ZuulKeyInfo):
def deserialize(self, data, context, extra=None):
data = super().deserialize(data, context, extra)
data['keys'] = [ZuulSecretKey.fromDict(k, extra['password_bytes'])
for k in data['keys']]
return data
def updateFromDict(self, context, extra=None, **data):
if 'keys' in data:
data['keys'] = [ZuulSecretKey.fromDict(k, extra['password_bytes'])
for k in data['keys']]
self.updateAttributes(context, **data)
@classmethod
def newFromDict(klass, context, extra=None, **data):
if 'keys' in data:
data['keys'] = [ZuulSecretKey.fromDict(k, extra['password_bytes'])
for k in data['keys']]
return super().new(context, **data)
def _handleKeys(cls, data, extra):
kp_manager = extra['keystore_password_manager']
keys = []
for keydata in data.get('keys', []):
key = None
error = None
for password in kp_manager.passwords_bytes:
try:
key = ZuulSecretKey.fromDict(keydata, password)
break
except ValueError as e:
error = e
if key is None:
raise error
if password != kp_manager.password_bytes:
data['_rekey'] = True
key._rekey(kp_manager.password_bytes)
keys.append(key)
data['keys'] = keys
class KeystorageCache(ZuulTreeCache):
def __init__(self, *args, **kw):
self.password = kw.pop('password')
self.password_bytes = self.password.encode('utf8')
self.extra = kw.pop('extra')
super().__init__(*args, **kw, auto_reinit=False)
def objectFromRaw(self, key, data, zstat):
extra = dict(password=self.password,
password_bytes=self.password_bytes)
if key.endswith('/ssh'):
info = ZuulSSHKeyInfo._fromRaw(
self._zk_context, data, zstat, extra)
self._zk_context, data, zstat, self.extra)
info._set(_path=key)
if getattr(info, '_rekey', False):
self.log.info("Re-encrypting %s with new password", key)
info.updateAttributes(self._zk_context)
return info
elif key.endswith('/secrets'):
info = ZuulSecretKeyInfo._fromRaw(
self._zk_context, data, zstat, extra)
self._zk_context, data, zstat, self.extra)
info._set(_path=key)
if getattr(info, '_rekey', False):
self.log.info("Re-encrypting %s with new password", key)
info.updateAttributes(self._zk_context)
return info
else:
raise Exception("Unknown key class")
def updateFromRaw(self, obj, key, data, zstat):
extra = dict(password=self.password,
password_bytes=self.password_bytes)
obj._updateFromRaw(self._zk_context, data, zstat, extra)
obj._updateFromRaw(self._zk_context, data, zstat, self.extra)
def parsePath(self, path):
key = None
@@ -299,13 +327,19 @@ class KeyStorage3(ZooKeeperBase):
log = logging.getLogger("zuul.KeyStorage")
ROOT_PATH = "/keystorage3"
def __init__(self, zookeeper_client, password, start_cache=True):
def __init__(self, zookeeper_client, keystore_password_manager,
start_cache=True):
super().__init__(zookeeper_client)
self.password = password
self.password_bytes = password.encode("utf-8")
self.kp_manager = keystore_password_manager
self.all_passwords = self.kp_manager.passwords
self.all_passwords_bytes = self.kp_manager.passwords_bytes
self.extra = dict(
keystore_password_manager=keystore_password_manager,
)
if start_cache:
self.keystorage_cache = KeystorageCache(
self.client, self.ROOT_PATH, password=self.password)
self.client, self.ROOT_PATH, extra=self.extra)
else:
self.keystorage_cache = None
@@ -326,8 +360,6 @@ class KeyStorage3(ZooKeeperBase):
def importKeys(self, import_data, overwrite):
self.keystorage_cache.ensureReady()
extra = dict(password=self.password,
password_bytes=self.password_bytes)
with self.createZKContext() as context:
for path, data in import_data['keys'].items():
if (path.startswith('/keystorage/') or
@@ -346,14 +378,15 @@ class KeyStorage3(ZooKeeperBase):
if existing := self.keystorage_cache._cached_objects.get(path):
if overwrite:
existing.updateFromDict(
context, extra=extra, **data)
context, extra=self.extra, **data)
self.log.info("Overwrote key at %s", path)
else:
self.log.warning(
"Not overwriting existing key at %s", path)
continue
else:
cls.newFromDict(context, extra=extra, _path=path, **data)
cls.newFromDict(context, extra=self.extra,
_path=path, **data)
self.log.info("Imported key at %s", path)
def getProjectSSHKeysPath(self, connection_name, project_name):
@@ -371,7 +404,7 @@ class KeyStorage3(ZooKeeperBase):
return None
self.log.info("Generating a new SSH key for %s/%s",
connection_name, project_name)
zuul_key = ZuulSSHKey.new(self.password, version=0)
zuul_key = ZuulSSHKey.new(self.kp_manager.password, version=0)
keys = [zuul_key]
try:
with self.createZKContext() as context:
@@ -395,7 +428,8 @@ class KeyStorage3(ZooKeeperBase):
connection_name, project_name)
keys = info.keys.copy()
version = len(keys)
zuul_key = ZuulSSHKey.new(self.password_bytes, version=version)
zuul_key = ZuulSSHKey.new(
self.kp_manager.password_bytes, version=version)
keys.append(zuul_key)
with self.createZKContext() as context:
info.updateAttributes(context, keys=keys)
@@ -456,7 +490,7 @@ class KeyStorage3(ZooKeeperBase):
return None
self.log.info("Generating a new SSH key for %s",
tenant_name)
zuul_key = ZuulSSHKey.new(self.password, version=0)
zuul_key = ZuulSSHKey.new(self.kp_manager.password, version=0)
keys = [zuul_key]
try:
with self.createZKContext() as context:
@@ -478,7 +512,8 @@ class KeyStorage3(ZooKeeperBase):
self.log.info("Generating a new SSH key for %s", tenant_name)
keys = info.keys.copy()
version = len(keys)
zuul_key = ZuulSSHKey.new(self.password_bytes, version=version)
zuul_key = ZuulSSHKey.new(
self.kp_manager.password_bytes, version=version)
keys.append(zuul_key)
with self.createZKContext() as context:
info.updateAttributes(context, keys=keys)
@@ -531,7 +566,8 @@ class KeyStorage3(ZooKeeperBase):
return None
self.log.info("Generating a new secrets key for %s/%s",
connection_name, project_name)
zuul_key = ZuulSecretKey.new(self.password_bytes, version=0)
zuul_key = ZuulSecretKey.new(
self.kp_manager.password_bytes, version=0)
keys = [zuul_key]
try:
with self.createZKContext() as context:
@@ -555,7 +591,8 @@ class KeyStorage3(ZooKeeperBase):
connection_name, project_name)
keys = info.keys.copy()
version = len(keys)
zuul_key = ZuulSecretKey.new(self.password_bytes, version=version)
zuul_key = ZuulSecretKey.new(
self.kp_manager.password_bytes, version=version)
keys.append(zuul_key)
with self.createZKContext() as context:
info.updateAttributes(context, keys=keys)
zuul/model.py
+49
View File
@@ -12345,3 +12345,52 @@ class TenantEventState(zkobject.ZKObject):
self._save(context, data, create=True)
except NodeExistsError:
self._save(context, data)
class KeystorePasswordManager:
def __init__(self, config):
try:
passwords = config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
passwords = [x.strip() for x in passwords.split('\n')]
self.password = None
self.password_bytes = None
self.passwords_by_hash = {}
# We rely on dict order; we will use the first password that
# all components have.
for password in passwords:
hasher = hashlib.sha256()
hasher.update(password.encode('utf8'))
pw_hash = hasher.hexdigest()
self.passwords_by_hash[pw_hash] = password
self.password_hashes = list(self.passwords_by_hash.keys())
self.passwords = passwords
self.passwords_bytes = [x.encode('utf8') for x in passwords]
def setPassword(self, common_hashes):
"Set the current password to the first common one"
for pw_hash, password in self.passwords_by_hash.items():
if pw_hash in common_hashes:
self.password = password
self.password_bytes = password.encode('utf8')
return
def connectToRegistry(self, component_registry):
component_registry.registerCallback(
lambda: self.registryCallback(component_registry))
self.registryCallback(component_registry)
def registryCallback(self, registry):
common_hashes = set()
for ctype, components in registry.all():
for component in components:
# Ignore components without passwords or the empty list
# (the component could be initializing).
if hashes := getattr(component, 'keystore_passwords', []):
if not common_hashes:
common_hashes = set(hashes)
else:
common_hashes &= set(hashes)
self.setPassword(common_hashes)
zuul/scheduler.py
+11 -8
View File
@@ -67,6 +67,7 @@ from zuul.model import (
HoldRequest,
ImageBuildArtifact,
ImageUpload,
KeystorePasswordManager,
MergeCompletedEvent,
NodeRequest,
NodesProvisionedEvent,
@@ -289,10 +290,18 @@ class Scheduler(threading.Thread):
self.zuul_version = get_version_string()
self.keystore_password_manager = KeystorePasswordManager(self.config)
self.component_info = SchedulerComponent(
self.zk_client, self.hostname, version=self.zuul_version)
self.zk_client, self.hostname, version=self.zuul_version,
keystore_passwords=self.keystore_password_manager.password_hashes,
)
self.component_info.register()
self.component_registry = COMPONENT_REGISTRY.create(self.zk_client)
# This is mostly for unit tests, where we can have an empty,
# initialized registry object.
self.component_registry.waitForComponent(self.component_info)
self.keystore_password_manager.connectToRegistry(
self.component_registry)
self.system_config_cache = SystemConfigCache(self.zk_client,
self.wake_event.set)
self.unparsed_config_cache = UnparsedConfigCache(self.zk_client)
@@ -337,7 +346,7 @@ class Scheduler(threading.Thread):
self.keystore = KeyStorage(
self.zk_client,
password=self._get_key_store_password())
self.keystore_password_manager)
self.abide = Abide()
self.unparsed_abide = UnparsedAbideConfig()
@@ -1327,12 +1336,6 @@ class Scheduler(threading.Thread):
result.wait()
log.debug("Enqueue complete")
def _get_key_store_password(self):
try:
return self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
def runTenantLayoutUpdates(self):
log = logging.getLogger("zuul.Scheduler.LayoutUpdate")
# Only run this after config priming is complete
zuul/web/__init__.py
+13 -8
View File
@@ -62,6 +62,7 @@ from zuul.model import (
DequeueEvent,
EnqueueEvent,
HoldRequest,
KeystorePasswordManager,
NodesetRequest,
PromoteEvent,
ProviderNode,
@@ -3947,8 +3948,12 @@ class ZuulWeb(object):
self.executor_api = ExecutorApi(self.zk_client, use_cache=False)
self.keystore_password_manager = KeystorePasswordManager(self.config)
self.component_info = WebComponent(
self.zk_client, self.hostname, version=get_version_string())
self.zk_client, self.hostname, version=get_version_string(),
keystore_passwords=self.keystore_password_manager.password_hashes,
)
self.component_info.register()
self.event_watcher = EventWatcher(self.zk_client, None)
@@ -3958,6 +3963,11 @@ class ZuulWeb(object):
self.monitoring_server.start()
self.component_registry = COMPONENT_REGISTRY.create(self.zk_client)
# This is mostly for unit tests, where we can have an empty,
# initialized registry object.
self.component_registry.waitForComponent(self.component_info)
self.keystore_password_manager.connectToRegistry(
self.component_registry)
self.system = ZuulSystem(self.zk_client)
self.system_config_thread = None
@@ -3968,7 +3978,8 @@ class ZuulWeb(object):
self.unparsed_config_cache = UnparsedConfigCache(self.zk_client)
self.keystore = KeyStorage(
self.zk_client, password=self._get_key_store_password())
self.zk_client,
self.keystore_password_manager)
self.globals = SystemAttributes.fromConfig(self.config)
self.ansible_manager = AnsibleManager(
default_version=self.globals.default_ansible_version)
@@ -4211,12 +4222,6 @@ class ZuulWeb(object):
self.repl.stop()
self.repl = None
def _get_key_store_password(self):
try:
return self.config["keystore"]["password"]
except KeyError:
raise RuntimeError("No key store password configured!")
def updateConfig(self):
while self._system_config_running:
try:
zuul/zk/components.py
+31 -2
View File
@@ -1,4 +1,5 @@
# Copyright 2020 BMW Group
# Copyright 2021-2026 Acme Gating, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@@ -151,17 +152,27 @@ class BaseComponent(ZooKeeperBase):
class SchedulerComponent(BaseComponent):
kind = "scheduler"
def __init__(self, *args, **kwargs):
keystore_passwords = kwargs.pop('keystore_passwords', [])
super().__init__(*args, **kwargs)
self.initial_state = {
"keystore_passwords": keystore_passwords
}
self.content.update(self.initial_state)
class ExecutorComponent(BaseComponent):
kind = "executor"
def __init__(self, *args, **kwargs):
keystore_passwords = kwargs.pop('keystore_passwords', [])
super().__init__(*args, **kwargs)
self.initial_state = {
"accepting_work": False,
"zone": None,
"allow_unzoned": False,
"process_merge_jobs": False,
"keystore_passwords": keystore_passwords,
}
self.content.update(self.initial_state)
@@ -186,6 +197,14 @@ class FingerGatewayComponent(BaseComponent):
class WebComponent(BaseComponent):
kind = "web"
def __init__(self, *args, **kwargs):
keystore_passwords = kwargs.pop('keystore_passwords', [])
super().__init__(*args, **kwargs)
self.initial_state = {
"keystore_passwords": keystore_passwords
}
self.content.update(self.initial_state)
class LauncherComponent(BaseComponent):
kind = "launcher"
@@ -245,6 +264,15 @@ class ComponentRegistry(ZooKeeperBase):
if self.client.connected:
self._onConnect()
def waitForComponent(self, wait_component):
while True:
for kind, components in self.all():
if wait_component.kind != kind:
continue
for component in components:
if component.hostname == wait_component.hostname:
return
def _getComponentRoot(self, kind):
return '/'.join([COMPONENTS_ROOT, kind])
@@ -344,7 +372,8 @@ class ComponentRegistry(ZooKeeperBase):
if kind is None:
return [(kind, list(components.values()))
for (kind, components) in self._cached_components.items()]
for (kind, components) in
list(self._cached_components.items())]
# Filter the cached components for the given kind
return self._cached_components.get(kind, {}).values()
@@ -400,7 +429,7 @@ class ComponentRegistry(ZooKeeperBase):
self.model_api = version
def _handleCallbacks(self):
for cb in self.callbacks:
for cb in list(self.callbacks):
try:
cb()
except Exception: