Pseudo-shard unique project names in keystore

There is a 4mb (by default) limit to the size of a packet which
can be sent to a zk client, which means with a typical repo name
length of around 25 chars we could store somewhere between
100,000 or 200,000 projects per connection.

An easy way to potentially increase that number is to shard by
path component, since most connections that would have that many
repos probably also have paths which would provide useful sharding.

This makes the keystore project path like:

  connection/prefix/project

Where prefix is the first path component. The project is still the
full project name with url quoting, so there is no issue with
collisions.

Some examples:

  project                -> project/project
  project/subproject     -> project/project%2Fsubproject
  project/sub/subproject -> project/project%2Fsub%2Fsubproject

Change-Id: I59a6e04a0520b65c4e5a5a87a38e2db0216ffc30

tests/unit/test_strings.py

@@ -0,0 +1,28 @@
+# Copyright 2021 Acme Gating, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from zuul.lib import strings
+
+from tests.base import BaseTestCase
+
+
+class TestStrings(BaseTestCase):
+
+    def test_unique_project_name(self):
+        self.assertEqual('project/project',
+                         strings.unique_project_name('project'))
+        self.assertEqual('project/project%2Fsubproject',
+                         strings.unique_project_name('project/subproject'))
+        self.assertEqual('project/project%2Fsub%2Fproject',
+                         strings.unique_project_name('project/sub/project'))

zuul/lib/keystorage.py

@@ -19,12 +19,11 @@ import logging
 import os
 import tempfile
 import time
-from urllib.parse import quote_plus
 
 import kazoo
 import paramiko
 
-from zuul.lib import encryption
+from zuul.lib import encryption, strings
 from zuul.zk import ZooKeeperBase
 
 RSA_KEY_SIZE = 2048
@@ -270,7 +269,7 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
         self.backup = backup
 
     def getProjectSSHKeys(self, connection_name, project_name):
-        key_project_name = quote_plus(project_name)
+        key_project_name = strings.unique_project_name(project_name)
         key_path = self.SSH_PATH.format(connection_name, key_project_name)
 
         try:
@@ -337,7 +336,7 @@ class ZooKeeperKeyStorage(ZooKeeperBase, KeyStorage):
         self.kazoo_client.create(key_path, value=data, makepath=True)
 
     def getProjectSecretsKeys(self, connection_name, project_name):
-        key_project_name = quote_plus(project_name)
+        key_project_name = strings.unique_project_name(project_name)
         key_path = self.SECRETS_PATH.format(connection_name, key_project_name)
 
         try:

zuul/lib/strings.py

@@ -0,0 +1,23 @@
+# Copyright 2021 Acme Gating, LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from urllib.parse import quote_plus
+
+
+def unique_project_name(project_name):
+    parts = project_name.split('/')
+    prefix = parts[0]
+
+    name = quote_plus(project_name)
+    return f'{prefix}/{name}'