Adds ability to join instances to existing Neutron networks

Also refactores networking code to move networking code from
Instance so that different types of network can be represented by
single simple interface

Implements: blueprint join-existing-neutron-networks
Change-Id: I90afcea6b1c12d7f6f564d8524f5c0d30e1b4686

meta/io.murano/Classes/resources/ExistingNeutronNetwork.yaml

@@ -0,0 +1,111 @@
+Namespaces:
+  =: io.murano.resources
+  std: io.murano
+  sys: io.murano.system
+
+Name: ExistingNeutronNetwork
+
+Extends: NeutronNetworkBase
+
+Properties:
+  internalNetworkName:
+    Contract: $.string()
+    Default: null
+    Usage: InOut
+
+  internalSubnetworkName:
+    Contract: $.string()
+    Default: null
+    Usage: InOut
+
+  externalNetworkName:
+    Contract: $.string()
+    Default: null
+    Usage: InOut
+
+Workflow:
+  initialize:
+    Body:
+      - $._netExplorer: new(sys:NetworkExplorer)
+      - $._networks: null
+      - $._subnetworks: null
+      - $._internalNetworkId: null
+      - $._internalSubnetworkId: null
+      - $._externalNetworkId: null
+
+  deploy:
+    Body:
+      - If: $.internalNetworkName = null
+        Then:
+          $.internalNetworkName: $._getNetworks().where(
+             $.get('router:external') = false).first().name
+
+      - If: $._internalNetworkId = null
+        Then:
+          $._internalNetworkId: $._getNetworks().where(
+            $.name = $this.internalNetworkName or
+            $.id = $this.internalNetworkName).first().id
+
+      - If: $.internalSubnetworkName = null
+        Then:
+          $.internalSubnetworkName: $._getSubnetworks().where(
+            $.network_id = $this._internalNetworkId).first().name
+      - If: $._internalSubnetworkId = null
+        Then:
+          $._internalSubnetworkId: $._getSubnetworks().where(
+            ($.name = $this.internalSubnetworkName or
+              $.id = $this.internalSubnetworkName) and
+                $.network_id = $this._internalNetworkId).first().id
+
+      - If: $.externalNetworkName = null
+        Then:
+          $.externalNetworkName: $._getNetworks().where(
+            $.get('router:external') = true).select($.name).firstOrDefault()
+      - If: $._externalNetworkId = null and $.externalNetworkName != null
+        Then:
+          $._externalNetworkId: $._getNetworks().where(
+            $.name = $this.externalNetworkName or
+            $.id = $this.externalNetworkName).first().id
+
+  _getNetworks:
+    Body:
+      - If: $._networks = null
+        Then:
+          $._networks: $._netExplorer.listNetworks()
+      - Return: $._networks
+
+  _getSubnetworks:
+    Body:
+      - If: $._subnetworks = null
+        Then:
+          $._subnetworks : $._netExplorer.listSubnetworks()
+      - Return: $._subnetworks
+
+  joinInstance:
+    Arguments:
+      - instance:
+          Contract: $.class(Instance).notNull()
+      - securityGroupName:
+          Contract: $.string()
+      - assignFloatingIp:
+          Contract: $.bool().notNull()
+      - sharedIps:
+          Contract:
+            - $.class(std:SharedIp)
+    Body:
+      - $fipName: null
+      - $floatingIpNeRef: null
+      - If: $assignFloatingIp
+        Then:
+          - $floatingIpNeRef: $._externalNetworkId
+          - $fipName: format('fip-{0}-{1}'. $.id(), $instance.name)
+
+      - Return: $.joinInstanceToNetwork(
+            instance => $instance,
+            securityGroupName => $securityGroupName,
+            sharedIps => $sharedIps,
+            netRef => $._internalNetworkId,
+            subnetRef => $._internalSubnetworkId,
+            floatingIpResourceName => $fipName,
+            floatingIpNeRef => $floatingIpNeRef
+          )

meta/io.murano/Classes/resources/Instance.yaml

@@ -58,6 +58,7 @@ Methods:
       - $.agent: new(sys:Agent, host => $)
       - $.resources: new(sys:Resources)
       - $.instanceTemplate: {}
+      - $._floatingIpOutputName: null
 
   # Called after the Instance template pieces are in place. It
   # is at this stage alterations to the template should be made
@@ -126,23 +127,25 @@ Methods:
       - $outputs: $.environment.stack.output()
       # Changing this to use the .networks attribute instead of 'addresses'
       - $.ipAddresses: $outputs.get(format('{0}-assigned-ips', $this.name)).values().flatten()
-      - $.floatingIpAddress: $outputs.get(format('{0}-FloatingIPaddress', $this.name))
+      - If: $._floatingIpOutputName != null
+        Then:
+          - $.floatingIpAddress: $outputs.get($._floatingIpOutputName)
       - $.environment.instanceNotifier.trackCloudInstance($this)
 
   detectPrimaryNetwork:
     Body:
-      - $.primaryNetwork: null
+      - $._primaryNetwork: null
       - If: $.networks.primaryNetwork != null
         Then:
-          - $.primaryNetwork: $.networks.primaryNetwork
+          - $._primaryNetwork: $.networks.primaryNetwork
         Else:
           - If: $.networks.useEnvironmentNetwork and $.environment.defaultNetworks.environment!=null
             Then:
-              - $.primaryNetwork: $.environment.defaultNetworks.environment
+              - $._primaryNetwork: $.environment.defaultNetworks.environment
             Else:
               - If: $.networks.useFlatNetwork and $.environment.defaultNetworks.flat!=null
                 Then:
-                  - $.primaryNetwork: $.environment.defaultNetworks.flat
+                  - $._primaryNetwork: $.environment.defaultNetworks.flat
 
   ensureNetworksDeployed:
     Body:
@@ -165,79 +168,35 @@ Methods:
       - securityGroupName:
           Contract: $.string()
     Body:
-      - If: $net = $.primaryNetwork
-        Then:
-          - $primary: true
-        Else:
-          - $primary: false
-
-      - If: $primary and $.assignFloatingIp and not $.getAttr(fipAssigned, false)
-        Then:
-          - $assignFip: true
-
-        Else:
-          - $assignFip: false
-
-      - $portname: $.name + '-port-to-' + $net.id()
-      - $netRef: { get_resource: $net.getNetworkName() }
-      - $subnetRef: { get_resource: $net.getSubnetName() }
-      - $template:
-          resources:
-            $portname:
-              type: 'OS::Neutron::Port'
-              properties:
-                network: $netRef
-                fixed_ips:
-                  - subnet: $subnetRef
-                security_groups:
-                  - get_resource: $securityGroupName
-                replacement_policy: AUTO
-            $.name:
-              properties:
-                networks:
-                  - port:
-                      get_resource: $portname
-      - $.instanceTemplate: $.instanceTemplate.mergeWith($template)
-
+      - $primary: $net = $._primaryNetwork
+      - $assignFip: $primary and $.assignFloatingIp and not $.getAttr(fipAssigned, false)
+      - $sharedIps: []
       - If: $primary
         Then:
-          - For: sip
-            In: $.sharedIps
-            Do:
-              - $template:
-                  resources:
-                    $portname:
-                      properties:
-                        allowed_address_pairs:
-                          - ip_address: $sip.getSharedIpRef()
-              - $.instanceTemplate: $.instanceTemplate.mergeWith($template)
-
-      - If: $assignFip
+          $sharedIps: $.sharedIps
+      - $joinResult: $net.joinInstance(
+            instance => $this,
+            securityGroupName => $securityGroupName,
+            assignFloatingIp => $assignFip,
+            sharedIps => $sharedIps
+          )
+      - $.instanceTemplate: $.instanceTemplate.mergeWith($joinResult.template)
+      
+      - If: $joinResult.portRef != null
         Then:
-          - $extNetId: $net.getExternalNetId()
-          - If: $extNetId != null
-            Then:
-              - $fip_name: $.name + '-FloatingIP-' + $net.id()
-              - $template:
-                  resources:
-                    $fip_name:
-                      type: 'OS::Neutron::FloatingIP'
-                      properties:
-                        floating_network: $extNetId
-                        port_id:
-                          get_resource: $portname
-                  outputs:
-                    $.name + '-FloatingIPaddress':
-                      value:
-                        get_attr: [$fip_name, floating_ip_address]
-                      description: Floating IP assigned
-              - If: $net.externalRouterId != null
-                Then:
-                  # (sjmc7) This is a workaround for https://bugs.launchpad.net/heat/+bug/1299259
-                  - $routerInterfaceName: $net.getRouterInterfaceName()
-                  - $template['resources'][$fip_name]['depends_on']: [$routerInterfaceName]
-              - $.instanceTemplate: $.instanceTemplate.mergeWith($template)
-              - $.setAttr(fipAssigned, true)
+          - $template:
+              resources:
+                $.name:
+                  properties:
+                    networks:
+                      - port:
+                          $joinResult.portRef
+          - $.instanceTemplate: $.instanceTemplate.mergeWith($template)
+      - $._floatingIpOutputName: coalesce($._floatingIpOutputName, $joinResult.instanceFipOutput)
+
+      - If: $assignFip and $joinResult.instanceFipOutput != null
+        Then:
+          - $.setAttr(fipAssigned, true)
 
   destroy:
     # FIXME(smelikyan): All allocated resources should be cleaned-up on destroy

meta/io.murano/Classes/resources/Network.yaml

@@ -1,11 +1,20 @@
 Namespaces:
-    =: io.murano.resources
+  =: io.murano.resources
+  std: io.murano
 
 Name: Network
 
 Methods:
-  getNetworkName:
-  getSubnetName:
-  getRouterInterfaceName:
-  getExternalNetId:
   deploy:
+
+  joinInstance:
+    Arguments:
+      - instance:
+          Contract: $.class(Instance).notNull()
+      - securityGroupName:
+          Contract: $.string()
+      - assignFloatingIp:
+          Contract: $.bool().notNull()
+      - sharedIps:
+          Contract:
+            - $.class(std:SharedIp)

meta/io.murano/Classes/resources/NeutronNetwork.yaml

@@ -5,7 +5,7 @@ Namespaces:
 
 Name: NeutronNetwork
 
-Extends: Network
+Extends: NeutronNetworkBase
 
 Properties:
   name:
@@ -38,8 +38,8 @@ Properties:
 Methods:
   initialize:
     Body:
-      - $.environment: $.find(std:Environment).require()
-      - $.netExplorer: new(sys:NetworkExplorer)
+      - $._environment: $.find(std:Environment).require()
+      - $._netExplorer: new(sys:NetworkExplorer)
 
   deploy:
     Body:
@@ -47,69 +47,111 @@ Methods:
         Then:
           - If: $.useDefaultDns and (not bool($.dnsNameserver))
             Then:
-              - $.dnsNameserver: $.netExplorer.getDefaultDns()
+              - $.dnsNameserver: $._netExplorer.getDefaultDns()
 
-          - $.createNetwork()
+          - $template: $._createNetwork()
           - If: $.autoUplink and (not bool($.externalRouterId))
             Then:
-              - $.externalRouterId: $.netExplorer.getDefaultRouter()
+              - $.externalRouterId: $._netExplorer.getDefaultRouter()
           - If: $.autogenerateSubnet and (not bool($.subnetCidr))
             Then:
-              - $.subnetCidr: $.netExplorer.getAvailableCidr($.externalRouterId, $.id())
-          - $.createSubnet()
-          - If: $.externalRouterId!=null
-            Then:
-              - $.createRouterInterface()
+              - $.subnetCidr: $._netExplorer.getAvailableCidr($.externalRouterId, $.id())
 
-          - $.environment.stack.push()
+          - $template: $template.mergeWith($._createSubnet())
+          - If: $.externalRouterId != null
+            Then:
+              - $template: $template.mergeWith($._createRouterInterface())
+
+          - $._environment.stack.updateTemplate($template)
+          - $._environment.stack.push()
           - $.setAttr(deployed, true)
 
-  createNetwork:
+  _createNetwork:
     Body:
-      - $template:
+      - Return:
           resources:
-            $.getNetworkName():
+            $._getNetworkName():
               type: 'OS::Neutron::Net'
               properties:
-                name: $.name
-      - $.environment.stack.updateTemplate($template)
+                name: format('{0}-{1}', $.name, $.id())
 
-  createSubnet:
+  _createSubnet:
     Body:
-      - $template:
+      - Return:
           resources:
-            $.getSubnetName():
+            $._getSubnetName():
               type: 'OS::Neutron::Subnet'
               properties:
-                network: { get_resource: $.getNetworkName() }
+                network: { get_resource: $._getNetworkName() }
                 ip_version: 4
                 dns_nameservers: [ $.dnsNameserver ]
                 cidr: $.subnetCidr
-      - $.environment.stack.updateTemplate($template)
 
-  createRouterInterface:
+  _createRouterInterface:
     Body:
-      - $template:
+      - Return:
           resources:
-            $.getRouterInterfaceName():
+            $._getRouterInterfaceName():
               type: 'OS::Neutron::RouterInterface'
               properties:
                 router_id: $.externalRouterId
-                subnet: { get_resource: $.getSubnetName() }
-      - $.environment.stack.updateTemplate($template)
+                subnet: { get_resource: $._getSubnetName() }
 
-  getRouterInterfaceName:
+  joinInstance:
+    Arguments:
+      - instance:
+          Contract: $.class(Instance).notNull()
+      - securityGroupName:
+          Contract: $.string()
+      - assignFloatingIp:
+          Contract: $.bool().notNull()
+      - sharedIps:
+          Contract:
+            - $.class(std:SharedIp)
     Body:
-      Return: $.name + '-ri-' + $.id()
+      - $netRef: { get_resource: $._getNetworkName() }
+      - $subnetRef: { get_resource: $._getSubnetName() }
+      - $extNetId: null
+      - $fipName: null
+      - If: $assignFloatingIp
+        Then:
+          - $extNetId: $._getExternalNetId()
+          - $fipName: format('fip-{0}-{1}'. $.id(), $instance.name)
 
-  getNetworkName:
-    Body:
-      Return: $.name + '-net-' + $.id()
+      - $result: $.joinInstanceToNetwork(
+            instance => $instance,
+            securityGroupName => $securityGroupName,
+            sharedIps => $sharedIps,
+            netRef => $netRef,
+            subnetRef => $subnetRef,
+            floatingIpResourceName => $fipName,
+            floatingIpNeRef => $extNetId
+          )
 
-  getSubnetName:
-    Body:
-      Return: $.name + '-subnet-' + $.id()
+      # (sjmc7) This is a workaround for https://bugs.launchpad.net/heat/+bug/1299259
+      - If: $externalRouterId != null
+        Then:
+          - $template:
+              resources:
+                $fipName:
+                  depends_on:
+                    - $._getRouterInterfaceName()
+          - $result.template: $result.template.mergeWith($template)
 
-  getExternalNetId:
+      - Return: $result
+
+  _getRouterInterfaceName:
     Body:
-      Return: $.netExplorer.getExternalNetworkIdForRouter($.externalRouterId)
+      Return: format('ri-{0}', $.id())
+
+  _getNetworkName:
+    Body:
+      Return: format('network-{0}', $.id())
+
+  _getSubnetName:
+    Body:
+      Return: format('subnet-{0}', $.id())
+
+  _getExternalNetId:
+    Body:
+      Return: $._netExplorer.getExternalNetworkIdForRouter($.externalRouterId)

meta/io.murano/Classes/resources/NeutronNetworkBase.yaml

@@ -0,0 +1,82 @@
+Namespaces:
+  =: io.murano.resources
+  std: io.murano
+
+Name: NeutronNetworkBase
+
+Extends: Network
+
+Methods:
+  joinInstanceToNetwork:
+    Arguments:
+      - instance:
+          Contract: $.class(Instance).notNull()
+      - securityGroupName:
+          Contract: $.string()
+      - sharedIps:
+          Contract:
+            - $.class(std:SharedIp)
+      - netRef:
+            Contract: $
+      - subnetRef:
+            Contract: $
+      - floatingIpResourceName:
+          Contract: $.string()
+      - floatingIpNeRef:
+          Contract: $
+    Body:
+      - $portName: format('port-{0}-{1}', $.id(), $instance.name)
+      - $patchTemplate:
+          resources:
+            $portName:
+              type: 'OS::Neutron::Port'
+              properties:
+                network: $netRef
+                fixed_ips:
+                  - subnet: $subnetRef
+                replacement_policy: AUTO
+
+      - If: bool($securityGroupName)
+        Then:
+          - $template:
+              resources:
+                $portName:
+                  properties:
+                    security_groups:
+                      - get_resource: $securityGroupName
+          - $patchTemplate: $patchTemplate.mergeWith($template)
+
+      - For: sip
+        In: $sharedIps
+        Do:
+          - $template:
+              resources:
+                $portName:
+                  properties:
+                    allowed_address_pairs:
+                      - ip_address: $sip.getSharedIpRef()
+          - $patchTemplate: $patchTemplate.mergeWith($template)
+
+      - $instanceFipOutput: null
+      - If: $floatingIpResourceName != null and $floatingIpNeRef != null
+        Then:
+          - $instanceFipOutput: $instance.name + '-floatingIPaddress'
+          - $template:
+              resources:
+                $floatingIpResourceName:
+                  type: 'OS::Neutron::FloatingIP'
+                  properties:
+                    floating_network: $floatingIpNeRef
+                    port_id:
+                      get_resource: $portName
+              outputs:
+                $instanceFipOutput:
+                  value:
+                    get_attr: [$floatingIpResourceName, floating_ip_address]
+                  description: format('Floating IP of {0}', $instance.name)
+          - $patchTemplate: $patchTemplate.mergeWith($template)
+      - Return:
+          template: $patchTemplate
+          portRef:
+            get_resource: $portName
+          instanceFipOutput: $instanceFipOutput

meta/io.murano/manifest.yaml

@@ -31,7 +31,9 @@ Classes:
   io.murano.resources.HeatSWConfigLinuxInstance: resources/HeatSWConfigLinuxInstance.yaml
   io.murano.resources.LinuxUDInstance: resources/LinuxUDInstance.yaml
   io.murano.resources.WindowsInstance: resources/WindowsInstance.yaml
+  io.murano.resources.NeutronNetworkBase: resources/NeutronNetworkBase.yaml
   io.murano.resources.NeutronNetwork: resources/NeutronNetwork.yaml
+  io.murano.resources.ExistingNeutronNetwork: resources/ExistingNeutronNetwork.yaml
 
   io.murano.system.Agent: system/Agent.yaml
   io.murano.system.AgentListener: system/AgentListener.yaml

murano/engine/system/net_explorer.py

@@ -165,3 +165,13 @@ class NetworkExplorer(murano_object.MuranoObject):
         net = netaddr.IPNetwork(
             '{0}/{1}'.format(self._settings.env_ip_template, mask_width))
         return list(net.subnet(width - bits_for_hosts))
+
+    # noinspection PyPep8Naming
+    def listNetworks(self, _context):
+        client = self._clients.get_neutron_client(_context)
+        return client.list_networks()['networks']
+
+    # noinspection PyPep8Naming
+    def listSubnetworks(self, _context):
+        client = self._clients.get_neutron_client(_context)
+        return client.list_subnets()['subnets']

murano/engine/system/yaql_functions.py

@@ -322,6 +322,27 @@ def _aggregate_with_seed(collection, selector, seed):
     return reduce(selector, collection, seed())
 
 
+@yaql.context.EvalArg('collection', collections.Iterable)
+def _first(collection):
+    return iter(collection).next()
+
+
+@yaql.context.EvalArg('collection', collections.Iterable)
+def _first_or_default(collection):
+    try:
+        return iter(collection).next()
+    except StopIteration:
+        return None
+
+
+@yaql.context.EvalArg('collection', collections.Iterable)
+def _first_or_default2(collection, default):
+    try:
+        return iter(collection).next()
+    except StopIteration:
+        return default
+
+
 def register(context):
     context.register_function(
         lambda json, mappings: _transform_json(json(), mappings()), 'bind')
@@ -359,3 +380,6 @@ def register(context):
     context.register_function(_take, 'take')
     context.register_function(_aggregate, 'aggregate')
     context.register_function(_aggregate_with_seed, 'aggregate')
+    context.register_function(_first, 'first')
+    context.register_function(_first_or_default, 'firstOrDefault')
+    context.register_function(_first_or_default2, 'firstOrDefault')