Integration Zuul and Nodepool into CI/CD App

* Using utility application ZuulNodepoolHelper
 * update test_cicd_apps

Change-Id: I4b0e093c0583b56a1b896d4304b71bcda42c8f0c

murano-apps/CiCdEnvironment/package/Classes/CiCdEnvironment.yaml

@@ -19,7 +19,15 @@ Properties:
   jenkins:
     Contract: $.class(ci_cd_pipeline_murano_app:Jenkins)
     Usage: Out
+  zuul:
+    Contract: $.class(ci_cd_pipeline_murano_app:Zuul)
+    Usage: Out
+  nodepool:
+    Contract: $.class(ci_cd_pipeline_murano_app:Nodepool)
+    Usage: Out
 
+  zuulNodepoolHelper:
+    Contract: $.class(ci_cd_pipeline_murano_app:ZuulNodepoolHelper)
   ldapRootUser:
     Contract: $.string().notNull()
   ldapRootPass:
@@ -122,18 +130,60 @@ Methods:
               gitUser => $.ldapUser,
               gitEmail => $.ldapEmail)
 
+      # Zuul
+      - If: $this.zuul = null
+        Then:
+          - $zuulInstanceName: concat($.instance_name, '-zuul')
+          - $zuulInstance: new(puppet:PuppetInstance, $._environment,
+              name => $zuulInstanceName,
+              flavor => $.flavor,
+              image => $.image,
+              keyname => $.keyname,
+              availabilityZone => $.availabilityZone,
+              assignFloatingIp => $.assignFloatingIp)
+          - $this.zuul: new(ci_cd_pipeline_murano_app:Zuul, $._environment,
+              name => 'Zuul',
+              instance => $zuulInstance,
+              gerrit => $this.gerrit)
+
+      # Nodepool
+      - If: $this.nodepool = null
+        Then:
+          - $nodepoolInstanceName: concat($.instance_name, '-nodepool')
+          - $nodepoolInstance: new(puppet:PuppetInstance, $._environment,
+              name => $nodepoolInstanceName,
+              flavor => $.flavor,
+              image => $.image,
+              keyname => $.keyname,
+              availabilityZone => $.availabilityZone,
+              assignFloatingIp => $.assignFloatingIp)
+          - $this.nodepool: new(ci_cd_pipeline_murano_app:Nodepool, $._environment,
+              name => 'Nodepool',
+              instance => $nodepoolInstance,
+              jenkins => $this.jenkins,
+              zuul => $this.zuul,
+              authUrl => $this.zuulNodepoolHelper.authUrl,
+              username => $this.zuulNodepoolHelper.username,
+              tenant => $this.zuulNodepoolHelper.tenant,
+              password => $this.zuulNodepoolHelper.password)
+
       # Deploy all
-      - Parallel:
+      - $applications:
-        - $this.ldap.deploy()
+          - $this.ldap
-        - $this.gerrit.deploy()
+          - $this.gerrit
-        - $this.jenkins.deploy()
+          - $this.jenkins
+
+      - If: $this.zuulNodepoolHelper != null
+        Then:
+          - $applications: $applications.append($this.zuul, $this.nodepool)
+
+      - $applications.pselect($.deploy())
 
       - $._environment.reporter.report($this, 'Adding hosts...')
 
       - $hosts: new(net:Hosts)
 
-      - $apps: [$.jenkins, $.gerrit, $.ldap]
+      - $applications.select($hosts.addHostByInstance($.instance))
-      - $apps.select($hosts.addHostByInstance($.instance))
+      - $applications.select($hosts.applyTo($.instance))
-      - $apps.select($hosts.applyTo($.instance))
 
       - $._environment.reporter.report($this, 'Hosts added.')

murano-apps/CiCdEnvironment/package/UI/ui.yaml

@@ -11,6 +11,7 @@ Application:
   ldapPass: $.appConfiguration.ldapPass
   ldapEmail: $.appConfiguration.ldapEmail
   userSSH: $.appConfiguration.userSSH
+  zuulNodepoolHelper: $.appConfiguration.zuulNodepoolHelper
   instance_name: generateHostname($.instanceConfiguration.unitNamingPattern, 1)
   flavor: $.instanceConfiguration.flavor
   image: $.instanceConfiguration.osImage
@@ -26,14 +27,14 @@ Forms:
           initial: root
           label: Root Username
           descriptionTitle: Root Username
-          description: >-
+          description: >
             Please, provide name for root user
         - name: ldapRootPass
           type: password
           initial: Openst@ck0
           label: Root Password
           descriptionTitle: Root Password
-          description: >-
+          description: >
             Please, provide password for root user
         - name: ldapRootEmail
           type: string
@@ -43,7 +44,7 @@ Forms:
             invalid: Please provide valid email address.
           label: Root User Email
           descriptionTitle: Root User Email
-          description: >-
+          description: >
             Please, provide email for root user
         - name: ldapUser
           type: string
@@ -51,7 +52,7 @@ Forms:
           label: Username
           required: false
           descriptionTitle: Regular User Name
-          description: >-
+          description: >
             Please, provide name for regular user
         - name: ldapPass
           type: password
@@ -59,7 +60,7 @@ Forms:
           label: Password
           required: false
           descriptionTitle: Regular User Password
-          description: >-
+          description: >
             Please, provide password for regular user
         - name: ldapEmail
           type: string
@@ -70,7 +71,7 @@ Forms:
           label: Email
           required: false
           descriptionTitle: Regular user Email
-          description: >-
+          description: >
             Please, provide regular user email
         - name: userSSH
           type: string
@@ -78,8 +79,15 @@ Forms:
           label: User ssh key
           required: false
           descriptionTitle: User public ssh key
-          description: >-
+          description: >
-            Please, provide user public ssh key
+            Please, provide user public ssh key.
+        - name: zuulNodepoolHelper
+          type: org.openstack.ci_cd_pipeline_murano_app.ZuulNodepoolHelper
+          label: Zuul and Nodepool
+          required: false
+          descriptionTitle: Zuul and Nodepool
+          description: >
+            Specify Zuul and Nodepool parameters if needed.
   - instanceConfiguration:
       fields:
         - name: title
@@ -90,7 +98,7 @@ Forms:
         - name: flavor
           type: flavor
           label: Instance flavor
-          description: >-
+          description: >
             Select registered in Openstack flavor. Consider that application performance
             depends on this parameter.
           initial: m1.medium
@@ -99,13 +107,13 @@ Forms:
           type: image
           imageType: linux
           label: Instance image
-          description: >-
+          description: >
             Select a valid image for the application. Image should already be prepared and
             registered in glance.
         - name: keyPair
           type: keypair
           label: Key Pair
-          description: >-
+          description: >
             Select a Key Pair to control access to instances. You can login to
             instances using this KeyPair after the deployment of application.
           required: false
@@ -123,6 +131,6 @@ Forms:
           errorMessages:
             invalid: Just letters, numbers, underscores and hyphens are allowed.
           helpText: Just letters, numbers, underscores and hyphens are allowed.
-          description: >-
+          description: >
             Specify a string, that will be used in instance hostname.
             Just A-Z, a-z, 0-9, dash and underline are allowed.

murano-apps/CiCdEnvironment/package/manifest.yaml

@@ -17,3 +17,6 @@ Require:
   org.openstack.ci_cd_pipeline_murano_app.Gerrit:
   org.openstack.ci_cd_pipeline_murano_app.puppet.Puppet:
   org.openstack.ci_cd_pipeline_murano_app.utils.CiCdUtils:
+  org.openstack.ci_cd_pipeline_murano_app.ZuulNodepoolHelper:
+  org.openstack.ci_cd_pipeline_murano_app.Zuul:
+  org.openstack.ci_cd_pipeline_murano_app.Nodepool:

murano-apps/ZuulNodepoolHelper/package/Classes/ZuulNodepoolHelper.yaml

@@ -0,0 +1,21 @@
+Namespaces:
+  =: org.openstack.ci_cd_pipeline_murano_app
+  std: io.murano
+  ci_cd_pipeline_murano_app: org.openstack.ci_cd_pipeline_murano_app
+
+Name: ZuulNodepoolHelper
+
+Extends: std:Application
+
+Properties:
+  name:
+    Contract: $.string().notNull()
+  # Openstack credentials.
+  authUrl:
+    Contract: $.string().notNull()
+  username:
+    Contract: $.string().notNull()
+  tenant:
+    Contract: $.string().notNull()
+  password:
+    Contract: $.string().notNull()

murano-apps/ZuulNodepoolHelper/package/UI/ui.yaml

@@ -0,0 +1,41 @@
+Version: 2
+
+Application:
+  ?:
+    type: org.openstack.ci_cd_pipeline_murano_app.ZuulNodepoolHelper
+  name: $.appConfiguration.name
+  authUrl: $.appConfiguration.authUrl
+  username: $.appConfiguration.username
+  tenant: $.appConfiguration.tenant
+  password: $.appConfiguration.password
+
+Forms:
+  - appConfiguration:
+      fields:
+        - name: name
+          type: string
+          label: Application Name
+          initial: Zuul and Nodepool Parameters
+          description: >
+            Enter a desired name for the application. Just A-Z, a-z, 0-9, dash and
+            underline are allowed
+        - name: authUrl
+          type: string
+          label: Keystone Auth URL
+          description: Provide Keystone Auth URL.
+          initial: http://192.168.0.2:5000/v2.0
+        - name: username
+          type: string
+          label: Keystone username
+          description: Provide Keystone username.
+          initial: admin
+        - name: tenant
+          type: string
+          label: Keystone tenant
+          description: Provide Keystone tenant name.
+          initial: admin
+        - name: password
+          type: string
+          description: Provide Keystone password.
+          label: Keystone password
+          initial: secret

murano-apps/ZuulNodepoolHelper/package/manifest.yaml

@@ -0,0 +1,12 @@
+Format: 1.0
+Type: Library
+FullName: org.openstack.ci_cd_pipeline_murano_app.ZuulNodepoolHelper
+Name: Zuul and Nodepool helper
+Description: |
+ Zuul and Nodepool Install Helper for CI/CD application.
+Author: 'Mirantis, Inc'
+Tags: [CI, helper]
+Classes:
+  org.openstack.ci_cd_pipeline_murano_app.ZuulNodepoolHelper: ZuulNodepoolHelper.yaml
+UI: ui.yaml
+Logo: logo.png

tests/base.py

@@ -71,6 +71,19 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
         # add such possibility
         self.os_cleanup_before = str2bool('OS_CLEANUP_BEFORE', False)
         self.os_cleanup_after = str2bool('OS_CLEANUP_AFTER', True)
+        #
+        self.os_username = os.environ.get('OS_USERNAME')
+        self.os_password = os.environ.get('OS_PASSWORD')
+        self.os_tenant_name = os.environ.get('OS_TENANT_NAME')
+        self.os_auth_uri = os.environ.get('OS_AUTH_URL')
+
+        # Data for Nodepool app
+        self.os_np_username = os.environ.get('OS_NP_USERNAME', self.os_username)
+        self.os_np_password = os.environ.get('OS_NP_PASSWORD', self.os_password)
+        self.os_np_tenant_name = os.environ.get('OS_NP_TENANT_NAME',
+                                                self.os_tenant_name)
+        self.os_np_auth_uri = os.environ.get('OS_NP_AUTH_URL', self.os_auth_uri)
+        self.os_np_cleanup_before = str2bool('OS_NP_CLEANUP_BEFORE', False)
 
         self.keystone = self.initialize_keystone_client()
         self.heat = self.initialize_heat_client(self.keystone)
@@ -82,16 +95,18 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
         self.envs = []
         if self.os_cleanup_before:
             self.cleanup_up_tenant()
+
+        if self.os_np_cleanup_before:
+            self.cleanup_up_np_tenant()
         LOG.info('Running test: {0}'.format(self._testMethodName))
 
     def tearDown(self):
-        if not self.os_cleanup_after:
+        if self.os_cleanup_after:
             for env in self.envs:
                 try:
                     self.delete_env(env)
                 except Exception:
                     self.delete_stack(env)
-
         super(MuranoTestsBase, self).tearDown()
 
     @staticmethod
@@ -121,6 +136,11 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
                 pass
         return
 
+    def cleanup_up_np_tenant(self):
+        # TODO
+        LOG.warning('NodePool cleanup not implemented yet!')
+        return
+
     def delete_stack(self, environment):
         stack = self._get_stack(environment.id)
         if not stack:
@@ -132,7 +152,8 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
         name = self.rand_name()
         environment = self.murano.environments.create({'name': name})
         self.envs.append(environment)
-        self.addCleanup(self.delete_env, environment)
+        if self.os_cleanup_after:
+            self.addCleanup(self.delete_env, environment)
         LOG.debug('Created Environment:\n {0}'.format(environment))
 
         return environment
@@ -308,7 +329,10 @@ class MuranoTestsBase(testtools.TestCase, clients.ClientsBase):
             'Deployment status is "{0}"'.format(deployment.state)
         )
 
-        fips = self.guess_fip(environment.services[0])
+        fips = {}
+
+        for service in environment.services:
+            fips.update(self.guess_fip(service))
 
         for service in services_map:
             LOG.debug(

tests/test_cicd_apps.py

@@ -21,7 +21,9 @@ class MuranoCiCdTest(base.MuranoTestsBase):
     def test_deploy_cicd(self):
         environment = self.create_env()
         session = self.create_session(environment)
-        service_json = {
+
+        zuul_helper_id = str(self.generate_id())
+        service_json1 = {
             '?': {
                 '_{id}'.format(id=self.generate_id().hex): {'name': 'CI/CD'},
                 'id': str(self.generate_id()),
@@ -40,9 +42,28 @@ class MuranoCiCdTest(base.MuranoTestsBase):
             'ldapRootPass': 'P@ssw0rd',
             'ldapRootUser': 'root',
             'ldapUser': 'user',
-            'name': 'CI/CD'
+            'name': 'CI/CD',
+            'zuulNodepoolHelper': zuul_helper_id
         }
-        self.create_service(environment, session, service_json)
+
+        service_json2 = {
+            '?': {
+                '_{id}'.format(id=self.generate_id().hex): {
+                    'name': 'Zuul and Nodepool helper'},
+                'id': zuul_helper_id,
+                'type':
+                    'org.openstack.ci_cd_pipeline_murano_app.ZuulNodepoolHelper'
+            },
+
+            'name': 'Zuul and Nodepool Parameters',
+            'authUrl': self.os_np_auth_uri,
+            'password': self.os_np_password,
+            'tenant': self.os_np_tenant_name,
+            'username': self.os_np_username
+        }
+
+        self.create_service(environment, session, service_json2)
+        self.create_service(environment, session, service_json1)
         self.deploy_env(environment, session)
 
         environment = self.get_env(environment)

tools/default_packages_list.sh

@@ -1 +1 @@
-export DEFAULT_PACKAGES_LIST="Puppet SystemConfig CiCdUtils OpenLDAP Gerrit Jenkins Zuul Nodepool CiCdEnvironment"
+export DEFAULT_PACKAGES_LIST="Puppet SystemConfig CiCdUtils OpenLDAP Gerrit Jenkins Zuul Nodepool ZuulNodepoolHelper CiCdEnvironment"

tox.ini

@@ -22,7 +22,9 @@ distribute = false
 commands = {posargs:}
 
 [testenv:deploy_cicd_apps]
-commands = python setup.py testr --testr-args='{posargs}'
+# FIXME!
+commands = python -m unittest tests.test_cicd_apps.MuranoCiCdTest.test_deploy_cicd
+#commands = python setup.py testr --testr-args='{posargs}'
 
 [testenv:hacking]
 deps=