Add CSAR validation and metadata extraction

Add validation of CSAR files as part of translation of CSAR files.
Also, add extraction of metadata from the CSAR files. Include
necessary unit tests.

Change-Id: I590ba71b1568a832c3b784ae99cce417a9b24f26
Partially-Implements: blueprint tosca-csar-translation

toscaparser/prereq/csar.py

@@ -0,0 +1,97 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os.path
+import yaml
+import zipfile
+
+from toscaparser.common.exception import ValidationError
+from toscaparser.utils.gettextutils import _
+
+
+class CSAR(object):
+
+    def __init__(self, csar_file):
+        self.csar_file = csar_file
+        self.is_validated = False
+
+    def validate(self):
+        """Validate the provided CSAR file."""
+
+        self.is_validated = True
+
+        # validate that the file exists
+        if not os.path.isfile(self.csar_file):
+            err_msg = (_('The file %s does not exist.') % self.csar_file)
+            raise ValidationError(message=err_msg)
+
+        # validate that it is a valid zip file
+        if not zipfile.is_zipfile(self.csar_file):
+            err_msg = (_('The file %s is not a valid zip file.')
+                       % self.csar_file)
+            raise ValidationError(message=err_msg)
+
+        # validate that it contains the metadata file in the correct location
+        self.zfile = zipfile.ZipFile(self.csar_file, 'r')
+        filelist = self.zfile.namelist()
+        if 'TOSCA-Metadata/TOSCA.meta' not in filelist:
+            err_msg = (_('The file %s is not a valid CSAR as it does not '
+                         'contain the required file "TOSCA.meta" in the '
+                         'folder "TOSCA-Metadata".') % self.csar_file)
+            raise ValidationError(message=err_msg)
+
+        # validate that 'Entry-Definitions' property exists in TOSCA.meta
+        data = self.zfile.read('TOSCA-Metadata/TOSCA.meta')
+        meta = yaml.load(data)
+        if type(meta) is not dict:
+            err_msg = (_('The file "TOSCA-Metadata/TOSCA.meta" in %s does not '
+                         'contain valid YAML content.') % self.csar_file)
+            raise ValidationError(message=err_msg)
+        self.metadata = meta
+
+        if 'Entry-Definitions' not in self.metadata:
+            err_msg = (_('The CSAR file "%s" is missing the required metadata '
+                         '"Entry-Definitions" in "TOSCA-Metadata/TOSCA.meta".')
+                       % self.csar_file)
+            raise ValidationError(message=err_msg)
+
+        # validate that 'Entry-Definitions' metadata value points to an
+        # existing file in the CSAR
+        entry = self.metadata['Entry-Definitions']
+        if entry not in filelist:
+            err_msg = (_('The "Entry-Definitions" file defined in the CSAR '
+                         '"%s" does not exist.') % self.csar_file)
+            raise ValidationError(message=err_msg)
+
+    def get_metadata(self):
+        """Return the metadata dictionary."""
+
+        # validate the csar if not already validated
+        if not self.is_validated:
+            self.validate()
+
+        # return a copy to avoid changes overwrite the original
+        return dict(self.metadata) if self.metadata else None
+
+    def _get_metadata(self, key):
+        if not self.is_validated:
+            self.validate()
+        return self.metadata[key] if key in self.metadata else None
+
+    def get_author(self):
+        return self._get_metadata('Created-By')
+
+    def get_version(self):
+        return self._get_metadata('CSAR-Version')
+
+    def get_main_template(self):
+        return self._get_metadata('Entry-Definitions')

toscaparser/tests/data/CSAR/csar_not_zip.zip

@@ -0,0 +1 @@
+This is an invalid CSAR file.

toscaparser/tests/test_prereq.py

@@ -0,0 +1,109 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from toscaparser.common.exception import ValidationError
+from toscaparser.prereq.csar import CSAR
+from toscaparser.tests.base import TestCase
+from toscaparser.utils.gettextutils import _
+
+
+class CSARPrereqTest(TestCase):
+
+    base_path = os.path.dirname(os.path.abspath(__file__))
+
+    def test_file_exists(self):
+        path = os.path.join(self.base_path, "data/CSAR/csar_not_there.zip")
+        csar = CSAR(path)
+        error = self.assertRaises(ValidationError, csar.validate)
+        self.assertEqual(_('The file %s does not exist.') % path, str(error))
+
+    def test_file_is_zip(self):
+        path = os.path.join(self.base_path, "data/CSAR/csar_not_zip.zip")
+        csar = CSAR(path)
+        error = self.assertRaises(ValidationError, csar.validate)
+        self.assertEqual(_('The file %s is not a valid zip file.') % path,
+                         str(error))
+
+    def test_metadata_file_exists(self):
+        path = os.path.join(self.base_path,
+                            "data/CSAR/csar_no_metadata_file.zip")
+        csar = CSAR(path)
+        error = self.assertRaises(ValidationError, csar.validate)
+        self.assertEqual(_('The file %s is not a valid CSAR as it does not '
+                           'contain the required file "TOSCA.meta" in the '
+                           'folder "TOSCA-Metadata".') % path, str(error))
+
+    def test_valid_metadata_file_exists(self):
+        path = os.path.join(self.base_path,
+                            "data/CSAR/csar_wrong_metadata_file.zip")
+        csar = CSAR(path)
+        error = self.assertRaises(ValidationError, csar.validate)
+        self.assertEqual(_('The file %s is not a valid CSAR as it does not '
+                           'contain the required file "TOSCA.meta" in the '
+                           'folder "TOSCA-Metadata".') % path, str(error))
+
+    def test_metadata_is_yaml(self):
+        path = os.path.join(self.base_path,
+                            "data/CSAR/csar_metadata_not_yaml.zip")
+        csar = CSAR(path)
+        error = self.assertRaises(ValidationError, csar.validate)
+        self.assertEqual(_('The file "TOSCA-Metadata/TOSCA.meta" in %s does '
+                           'not contain valid YAML content.') % path,
+                         str(error))
+
+    def test_metadata_exists(self):
+        path = os.path.join(self.base_path,
+                            "data/CSAR/csar_missing_metadata.zip")
+        csar = CSAR(path)
+        error = self.assertRaises(ValidationError, csar.validate)
+        self.assertEqual(_('The CSAR file "%s" is missing the required '
+                           'metadata "Entry-Definitions" in '
+                           '"TOSCA-Metadata/TOSCA.meta".') % path, str(error))
+
+    def test_entry_def_exists(self):
+        path = os.path.join(self.base_path,
+                            "data/CSAR/csar_invalid_entry_def.zip")
+        csar = CSAR(path)
+        error = self.assertRaises(ValidationError, csar.validate)
+        self.assertEqual(_('The "Entry-Definitions" file defined in the CSAR '
+                           '"%s" does not exist.') % path, str(error))
+
+    def test_valid_csar(self):
+        path = os.path.join(self.base_path, "data/CSAR/csar_hello_world.zip")
+        csar = CSAR(path)
+        self.assertIsNone(csar.validate())
+
+    def test_metadata_invalid_csar(self):
+        path = os.path.join(self.base_path,
+                            "data/CSAR/csar_metadata_not_yaml.zip")
+        csar = CSAR(path)
+        error = self.assertRaises(ValidationError, csar.get_author)
+        self.assertEqual(_('The file "TOSCA-Metadata/TOSCA.meta" in %s does '
+                           'not contain valid YAML content.') % path,
+                         str(error))
+
+    def test_metadata_valid_csar(self):
+        path = os.path.join(self.base_path, "data/CSAR/csar_hello_world.zip")
+        csar = CSAR(path)
+        expected_meta = {'TOSCA-Meta-File-Version': 1.0,
+                         'CSAR-Version': 1.1,
+                         'Created-By': 'OASIS TOSCA TC',
+                         'Entry-Definitions': 'tosca_helloworld.yaml'}
+        self.assertEqual(expected_meta, csar.get_metadata(),
+                         'The extracted metadata of the CSAR file %(csar)s '
+                         'does not match the expected metadata %(meta)s'
+                         % {'csar': path, 'meta': expected_meta.__str__()})
+        self.assertEqual(1.1, csar.get_version())
+        self.assertEqual('OASIS TOSCA TC', csar.get_author())
+        self.assertEqual('tosca_helloworld.yaml', csar.get_main_template())