Browse Source

Merge "Add caching and cleanup of chart tarballs"

Zuul 5 months ago
parent
commit
d35896537b

+ 28
- 26
armada/handlers/armada.py View File

@@ -103,7 +103,7 @@ class Armada(object):
103 103
             raise
104 104
         self.manifest = Manifest(
105 105
             self.documents, target_manifest=target_manifest).get_manifest()
106
-        self.cloned_dirs = set()
106
+        self.chart_cache = {}
107 107
         self.chart_deploy = ChartDeploy(
108 108
             disable_update_pre, disable_update_post, self.dry_run,
109 109
             k8s_wait_attempts, k8s_wait_attempt_sleep, timeout, self.tiller)
@@ -119,16 +119,12 @@ class Armada(object):
119 119
             raise tiller_exceptions.TillerServicesUnavailableException()
120 120
 
121 121
         # Clone the chart sources
122
-        repos = {}
123 122
         manifest_data = self.manifest.get(const.KEYWORD_ARMADA, {})
124 123
         for group in manifest_data.get(const.KEYWORD_GROUPS, []):
125 124
             for ch in group.get(const.KEYWORD_CHARTS, []):
126
-                self.tag_cloned_repo(ch, repos)
125
+                self.get_chart(ch)
127 126
 
128
-                for dep in ch.get('chart', {}).get('dependencies', []):
129
-                    self.tag_cloned_repo(dep, repos)
130
-
131
-    def tag_cloned_repo(self, ch, repos):
127
+    def get_chart(self, ch):
132 128
         chart = ch.get('chart', {})
133 129
         chart_source = chart.get('source', {})
134 130
         location = chart_source.get('location')
@@ -138,25 +134,30 @@ class Armada(object):
138 134
         if ct_type == 'local':
139 135
             chart['source_dir'] = (location, subpath)
140 136
         elif ct_type == 'tar':
141
-            LOG.info('Downloading tarball from: %s', location)
137
+            source_key = (ct_type, location)
142 138
 
143
-            if not CONF.certs:
144
-                LOG.warn('Disabling server validation certs to extract charts')
145
-                tarball_dir = source.get_tarball(location, verify=False)
146
-            else:
147
-                tarball_dir = source.get_tarball(location, verify=CONF.cert)
139
+            if source_key not in self.chart_cache:
140
+                LOG.info('Downloading tarball from: %s', location)
148 141
 
149
-            chart['source_dir'] = (tarball_dir, subpath)
142
+                if not CONF.certs:
143
+                    LOG.warn(
144
+                        'Disabling server validation certs to extract charts')
145
+                    tarball_dir = source.get_tarball(location, verify=False)
146
+                else:
147
+                    tarball_dir = source.get_tarball(
148
+                        location, verify=CONF.cert)
149
+                self.chart_cache[source_key] = tarball_dir
150
+            chart['source_dir'] = (self.chart_cache.get(source_key), subpath)
150 151
         elif ct_type == 'git':
151 152
             reference = chart_source.get('reference', 'master')
152
-            repo_branch = (location, reference)
153
+            source_key = (ct_type, location, reference)
153 154
 
154
-            if repo_branch not in repos:
155
+            if source_key not in self.chart_cache:
155 156
                 auth_method = chart_source.get('auth_method')
156 157
                 proxy_server = chart_source.get('proxy_server')
157 158
 
158 159
                 logstr = 'Cloning repo: {} from branch: {}'.format(
159
-                    *repo_branch)
160
+                    location, reference)
160 161
                 if proxy_server:
161 162
                     logstr += ' proxy: {}'.format(proxy_server)
162 163
                 if auth_method:
@@ -164,19 +165,20 @@ class Armada(object):
164 165
                 LOG.info(logstr)
165 166
 
166 167
                 repo_dir = source.git_clone(
167
-                    *repo_branch,
168
+                    location,
169
+                    reference,
168 170
                     proxy_server=proxy_server,
169 171
                     auth_method=auth_method)
170
-                self.cloned_dirs.add(repo_dir)
171 172
 
172
-                repos[repo_branch] = repo_dir
173
-                chart['source_dir'] = (repo_dir, subpath)
174
-            else:
175
-                chart['source_dir'] = (repos.get(repo_branch), subpath)
173
+                self.chart_cache[source_key] = repo_dir
174
+            chart['source_dir'] = (self.chart_cache.get(source_key), subpath)
176 175
         else:
177 176
             chart_name = chart.get('chart_name')
178 177
             raise source_exceptions.ChartSourceException(ct_type, chart_name)
179 178
 
179
+        for dep in ch.get('chart', {}).get('dependencies', []):
180
+            self.get_chart(dep)
181
+
180 182
     def sync(self):
181 183
         '''
182 184
         Synchronize Helm with the Armada Config(s)
@@ -293,9 +295,9 @@ class Armada(object):
293 295
         LOG.info("Performing post-flight operations.")
294 296
 
295 297
         # Delete temp dirs used for deployment
296
-        for cloned_dir in self.cloned_dirs:
297
-            LOG.debug('Removing cloned temp directory: %s', cloned_dir)
298
-            source.source_cleanup(cloned_dir)
298
+        for chart_dir in self.chart_cache.values():
299
+            LOG.debug('Removing temp chart directory: %s', chart_dir)
300
+            source.source_cleanup(chart_dir)
299 301
 
300 302
     def _chart_cleanup(self, prefix, charts, msg):
301 303
         LOG.info('Processing chart cleanup to remove unspecified releases.')

+ 1
- 0
armada/tests/unit/handlers/test_armada.py View File

@@ -152,6 +152,7 @@ CHART_SOURCES = [('git://github.com/dummy/armada', 'chart_1'),
152 152
                  ('/tmp/dummy/armada', 'chart_4')]
153 153
 
154 154
 
155
+# TODO(seaneagan): Add unit tests with dependencies, including transitive.
155 156
 class ArmadaHandlerTestCase(base.ArmadaTestCase):
156 157
 
157 158
     def _test_pre_flight_ops(self, armada_obj):

+ 2
- 21
armada/tests/unit/utils/test_source.py View File

@@ -15,7 +15,6 @@
15 15
 import os
16 16
 import shutil
17 17
 
18
-import fixtures
19 18
 import mock
20 19
 import testtools
21 20
 
@@ -156,23 +155,6 @@ class GitTestCase(base.ArmadaTestCase):
156 155
         source.source_cleanup(git_path)
157 156
         mock_log.warning.assert_not_called()
158 157
 
159
-    @test_utils.attr(type=['negative'])
160
-    @mock.patch.object(source, 'LOG')
161
-    @mock.patch('armada.utils.source.shutil')
162
-    def test_source_cleanup_fake_git_path(self, mock_shutil, mock_log):
163
-        # Verify that passing in a fake path does nothing but log a warning.
164
-        # Don't want anyone using the function to delete random directories.
165
-        fake_path = self.useFixture(fixtures.TempDir()).path
166
-        self.addCleanup(shutil.rmtree, fake_path)
167
-        source.source_cleanup(fake_path)
168
-
169
-        mock_shutil.rmtree.assert_not_called()
170
-        self.assertTrue(mock_log.warning.called)
171
-        actual_call = mock_log.warning.mock_calls[0][1][:2]
172
-        self.assertEqual(
173
-            ('%s is not a valid git repository. Details: %s', fake_path),
174
-            actual_call)
175
-
176 158
     @test_utils.attr(type=['negative'])
177 159
     @mock.patch.object(source, 'LOG')
178 160
     @mock.patch('armada.utils.source.shutil')
@@ -187,9 +169,8 @@ class GitTestCase(base.ArmadaTestCase):
187 169
         mock_shutil.rmtree.assert_not_called()
188 170
         self.assertTrue(mock_log.warning.called)
189 171
         actual_call = mock_log.warning.mock_calls[0][1]
190
-        self.assertEqual(
191
-            ('Could not delete the path %s. Is it a git repository?', path),
192
-            actual_call)
172
+        self.assertEqual(('Could not find the chart path %s to delete.', path),
173
+                         actual_call)
193 174
 
194 175
     @testtools.skipUnless(base.is_connected(),
195 176
                           'git clone requires network connectivity.')

+ 10
- 18
armada/utils/source.py View File

@@ -153,27 +153,19 @@ def extract_tarball(tarball_path):
153 153
     return temp_dir
154 154
 
155 155
 
156
-def source_cleanup(git_path):
157
-    '''Clean up the git repository that was created by ``git_clone`` above.
156
+def source_cleanup(chart_path):
157
+    '''Clean up the chart path that was created above.
158 158
 
159
-    Removes the ``git_path`` repository and all associated files if they
159
+    Removes the ``chart_path`` and all associated files if they
160 160
     exist.
161 161
 
162
-    :param str git_path: The git repository to delete.
162
+    :param str chart_path: The chart path to delete.
163 163
     '''
164
-    if os.path.exists(git_path):
164
+    if os.path.exists(chart_path):
165 165
         try:
166
-            # Internally validates whether the path points to an actual repo.
167
-            Repo(git_path)
168
-        except git_exc.InvalidGitRepositoryError as e:
169
-            LOG.warning('%s is not a valid git repository. Details: %s',
170
-                        git_path, e)
171
-        else:
172
-            try:
173
-                shutil.rmtree(git_path)
174
-            except OSError as e:
175
-                LOG.warning('Could not delete the path %s. Details: %s',
176
-                            git_path, e)
166
+            shutil.rmtree(chart_path)
167
+        except OSError as e:
168
+            LOG.warning('Could not delete the path %s. Details: %s',
169
+                        chart_path, e)
177 170
     else:
178
-        LOG.warning('Could not delete the path %s. Is it a git repository?',
179
-                    git_path)
171
+        LOG.warning('Could not find the chart path %s to delete.', chart_path)

Loading…
Cancel
Save