Added CLI command to import packages

Added CLI command to import Murano packages. Example
murano-manage import-package murano-api/meta/packages/core

Also updated base packages structure.

Change-Id: I82c26ee6d8c675563174cefaa84b44ebdf1b14c4

contrib/devstack/lib/murano

@@ -133,7 +133,9 @@ function init_murano() {
     # (re)create Murano database
     recreate_database murano utf8
 
-    $MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE db_sync
+    $MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE db-sync
+    $MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE import-package $MURANO_DIR/meta/packages/core
+    $MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE import-package $MURANO_DIR/meta/packages/activedirectory
 }
 
 

meta/packages/activedirectory/Classes/ad.yaml

@@ -0,0 +1,33 @@
+Namespaces:
+  =: io.murano.services.windows.activeDirectory
+  std: io.murano
+  sys: io.murano.system
+  win: io.murano.services.windows
+
+Name: ActiveDirectory
+
+Extends: std:Application
+
+Properties:
+  name:
+    Contract: $.string().notNull()
+
+  primaryController:
+    Contract: $.class(PrimaryController).notNull()
+
+  secondaryControllers:
+    Contract: [$.class(SecondaryController).notNull()]
+
+  adminAccountName:
+    Contract: $.string().notNull()
+    Default: Administrator
+
+  adminPassword:
+    Contract: $.string().notNull()
+    Default: P@ssw0rd
+
+Workflow:
+  deploy:
+    Body:
+      -  $.primaryController.deploy()
+      -  $.secondaryControllers.pselect($.deploy())

meta/packages/activedirectory/Classes/controller.yaml

@@ -0,0 +1,19 @@
+Namespaces:
+    =: io.murano.services.windows.activeDirectory
+    std: io.murano
+    sys: io.murano.system
+    win: io.murano.services.windows
+
+Name: Controller
+
+Properties:
+  host:
+    Contract: $.class(win:Host).notNull()
+
+  recoveryPassword:
+    Contract: $.string().notNull()
+    Default: P@ssw0rd
+
+Workflow:
+  deploy:
+    Body: $.host.deploy()

meta/packages/activedirectory/Classes/primary_controller.yaml

@@ -0,0 +1,35 @@
+Namespaces:
+    =: io.murano.services.windows.activeDirectory
+    std: io.murano
+    sys: io.murano.system
+    win: io.murano.services.windows
+
+Name: PrimaryController
+
+Extends: Controller
+
+Properties:
+
+  dnsIp:
+    Contract: $.string()
+
+Workflow:
+  initialize:
+    Body:
+      - $.super($.initialize())
+      - $.domain: $.find(ActiveDirectory).require()
+
+  deploy:
+    Arguments:
+    Body:
+      - $.debugPrint('Deploying Primary Controller for domain {0}'.format($this.domain.name))
+      - $.super($.deploy())
+      - $resources: new(io.murano.system.Resources)
+      - $template: $resources.json('CreatePrimaryDC.template').bind(dict(
+            domain => $.domain.name,
+            recoveryPassword => $.recoveryPassword
+          ))
+      - $.host.agent.call($template, $resources)
+
+      - $template: $resources.json('AskDnsIp.template')
+      - $.dnsIp: $.host.agent.call($template, $resources)[0]

meta/packages/activedirectory/Classes/secondary_controller.yaml

@@ -0,0 +1,29 @@
+Namespaces:
+  =: io.murano.services.windows.activeDirectory
+  std: io.murano
+  sys: io.murano.system
+
+Name: SecondaryController
+
+Extends: Controller
+
+Workflow:
+  initialize:
+    Body:
+      - $.super($.initialize())
+      - $.domain: $.find(ActiveDirectory).require()
+
+  deploy:
+    Body:
+      - $.debugPrint('Deploying Secondary Controller for domain {0}'.format($this.domain.name))
+      - $.super($.deploy())
+      - $.host.joinDomain($.domain)
+      - $resources: new(sys:Resources)
+      - $template: $resources.json('CreateSecondaryDC.template').bind(dict(
+            domain => $.domain.name,
+            recoveryPassword => $.recoveryPassword,
+            domainAccountName => $.domain.adminAccountName,
+            domainPassword => $.domain.adminPassword
+          ))
+      - $.host.agent.call($template, $resources)
+#

meta/packages/activedirectory/Resources/AskDnsIp.template

@@ -0,0 +1,12 @@
+{
+  "Scripts": [
+    "Get-DnsListeningIpAddress.ps1"
+  ],
+  "Commands": [
+    {
+      "Name": "Get-DnsListeningIpAddress",
+      "Arguments": {}
+    }
+  ],
+  "RebootOnCompletion": 0
+}

meta/packages/activedirectory/Resources/CreatePrimaryDC.template

@@ -0,0 +1,16 @@
+{
+  "Scripts": [
+    "ImportCoreFunctions.ps1",
+    "Install-RolePrimaryDomainController.ps1"
+  ],
+  "Commands": [
+      {
+          "Name": "Install-RolePrimaryDomainController",
+          "Arguments": {
+          "DomainName": "$domain",
+              "SafeModePassword": "$recoveryPassword"
+          }
+      }
+  ],
+  "RebootOnCompletion": 1
+}

meta/packages/activedirectory/Resources/CreateSecondaryDC.template

@@ -0,0 +1,18 @@
+{
+  "Scripts": [
+    "ImportCoreFunctions.ps1",
+    "Install-RoleSecondaryDomainController.ps1"
+  ],
+  "Commands": [
+    {
+      "Name": "Install-RoleSecondaryDomainController",
+      "Arguments": {
+        "DomainName": "$domain",
+        "UserName": "$domainAccountName",
+        "Password": "$domainPassword",
+        "SafeModePassword": "$recoveryPassword"
+      }
+    }
+  ],
+  "RebootOnCompletion": 1
+}

meta/packages/activedirectory/Resources/scripts/Get-DnsListeningIpAddress.ps1

@@ -0,0 +1,7 @@
+
+function Get-DnsListeningIpAddress {
+    Import-Module DnsServer
+
+    (Get-DNSServer -ComputerName localhost).ServerSetting.ListeningIpAddress |
+        Where-Object { $_ -match "\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}" }
+}

meta/packages/activedirectory/Resources/scripts/ImportCoreFunctions.ps1

@@ -0,0 +1,68 @@
+
+Import-Module CoreFunctions -Force
+Initialize-Logger 'MuranoAgent' 'C:\Murano\PowerShell.log'
+
+
+function Show-InvocationInfo {
+    param (
+        $Invocation,
+        [Switch] $End
+    )
+
+    if ($End) {
+        Write-LogDebug "</function name='$($Invocation.MyCommand.Name)'>"
+    }
+    else {
+        Write-LogDebug "<function name='$($Invocation.MyCommand.Name)'>"
+        Write-LogDebug "<param>"
+        foreach ($Parameter in $Invocation.MyCommand.Parameters) {
+            foreach ($Key in $Parameter.Keys) {
+                $Type = $Parameter[$Key].ParameterType.FullName
+                foreach ($Value in $Invocation.BoundParameters[$Key]) {
+                    Write-LogDebug "[$Type] $Key = '$Value'"
+                }
+            }
+        }
+        Write-LogDebug "</param>"
+    }
+}
+
+
+$TrapHandler = {
+    Write-LogError "<exception>"
+    Write-LogError $_ -EntireObject
+    Write-LogError "</exception>"
+    break
+}
+
+
+trap {
+    &$TrapHandler
+}
+
+$ErrorActionPreference = 'Stop'
+
+
+<#
+# Usage example for Show-InvocationInfo
+
+function MyFunction {
+    param (
+        [String] $Value1,
+        [String] $Value2,
+        [Int] $Int1
+    )
+    begin {
+        Show-InvocationInfo $MyInvocation
+    }
+    end {
+        Show-InvocationInfo $MyInvocation -End
+    }
+    process {
+        trap {
+            &$TrapHandler
+        }
+        # Main code here
+    }
+}
+#>

meta/packages/activedirectory/Resources/scripts/Install-RolePrimaryDomainController.ps1

@@ -0,0 +1,43 @@
+
+trap {
+    &$TrapHandler
+}
+
+
+Function Install-RolePrimaryDomainController {
+	param (
+		[String] $DomainName,
+		[String] $SafeModePassword
+	)
+    begin {
+        Show-InvocationInfo $MyInvocation
+    }
+    end {
+        Show-InvocationInfo $MyInvocation -End
+    }
+    process {
+        trap {
+            &$TrapHandler
+        }
+
+		Add-WindowsFeatureWrapper `
+			-Name "DNS","AD-Domain-Services","RSAT-DFS-Mgmt-Con" `
+			-IncludeManagementTools `
+	        -NotifyRestart
+
+		Write-Log "Creating first domain controller ..."
+			
+		$SMAP = ConvertTo-SecureString -String $SafeModePassword -AsPlainText -Force
+			
+		$null = Install-ADDSForest `
+			-DomainName $DomainName `
+			-SafeModeAdministratorPassword $SMAP `
+			-DomainMode Default `
+			-ForestMode Default `
+			-NoRebootOnCompletion `
+			-Force
+
+		Write-Log "Waiting 60 seconds for reboot ..."
+		Start-Sleep -Seconds 60
+	}
+}

meta/packages/activedirectory/Resources/scripts/Install-RoleSecondaryDomainController.ps1

@@ -0,0 +1,69 @@
+
+trap {
+    &$TrapHandler
+}
+
+
+Function Install-RoleSecondaryDomainController
+{
+<#
+.SYNOPSIS
+Install additional (secondary) domain controller.
+
+#>
+	param
+	(
+		[String]
+		# Domain name to join to.
+		$DomainName,
+		
+		[String]
+		# Domain user who is allowed to join computer to domain.
+		$UserName,
+		
+		[String]
+		# User's password.
+		$Password,
+		
+		[String]
+		# Domain controller recovery mode password.
+		$SafeModePassword
+	)
+    begin {
+        Show-InvocationInfo $MyInvocation
+    }
+    end {
+        Show-InvocationInfo $MyInvocation -End
+    }
+    process {
+        trap {
+            &$TrapHandler
+        }
+	
+		$Credential = New-Credential -UserName "$DomainName\$UserName" -Password $Password
+			
+		# Add required windows features
+		Add-WindowsFeatureWrapper `
+			-Name "DNS","AD-Domain-Services","RSAT-DFS-Mgmt-Con" `
+			-IncludeManagementTools `
+	                -NotifyRestart
+			
+		
+	    Write-Log "Adding secondary domain controller ..."
+	    
+		$SMAP = ConvertTo-SecureString -String $SafeModePassword -AsPlainText -Force
+
+		Install-ADDSDomainController `
+			-DomainName $DomainName `
+			-SafeModeAdministratorPassword $SMAP `
+			-Credential $Credential `
+			-NoRebootOnCompletion `
+			-Force `
+			-ErrorAction Stop | Out-Null
+
+		Write-Log "Waiting for restart ..."
+	#	Stop-Execution -ExitCode 3010 -ExitString "Computer must be restarted to finish domain controller promotion."
+	#	Write-Log "Restarting computer ..."
+	#	Restart-Computer -Force
+	}
+}

meta/packages/activedirectory/UI/ui.yaml

@@ -0,0 +1,143 @@
+unitTemplates:
+  - isMaster: true
+    recoveryPassword: {YAQL: $.serviceConfiguration.recoveryPassword}
+  - isMaster: false
+    recoveryPassword: {YAQL: $.serviceConfiguration.recoveryPassword}
+
+forms:
+  - serviceConfiguration:
+      fields:
+        - name: configuration
+          type: string
+          hidden: true
+          initial: standalone
+        - name: name
+          type: string
+          label: Domain Name
+          description: >-
+            Enter a desired name for a new domain. This name should fit to
+            DNS Domain Name requirements: it should contain
+            only A-Z, a-z, 0-9, (.) and (-) and should not end with a dash.
+            DNS server will be automatically set up on each of the Domain 
+            Controller instances. Note: Only first 15 characters or characters
+            before first period is used as NetBIOS name.
+          attributeNames: [name, domain]
+          minLength: 2
+          maxLength: 255
+          validators:
+            - expr:
+                regexpValidator: '^([0-9A-Za-z]|[0-9A-Za-z][0-9A-Za-z-]*[0-9A-Za-z])\.[0-9A-Za-z][0-9A-Za-z-]*[0-9A-Za-z]$'
+              message: >-
+                Only letters, numbers and dashes in the middle are
+                allowed. Period characters are allowed only when they
+                are used to delimit the components of domain style
+                names. Single-level domain is not
+                appropriate. Subdomains are not allowed.
+            - expr:
+                regexpValidator: '(^[^.]+$|^[^.]{1,15}\..*$)'
+              message: >-
+                NetBIOS name cannot be shorter than 1 symbol and
+                longer than 15 symbols.
+            - expr:
+                regexpValidator: '(^[^.]+$|^[^.]*\.[^.]{2,63}.*$)'
+              message: >-
+                DNS host name cannot be shorter than 2 symbols and
+                longer than 63 symbols.
+          helpText: >-
+            Just letters, numbers and dashes are allowed.
+            A dot can be used to create subdomains
+        - name: dcInstances
+          type: instance
+          label: Instance Count
+          description: >-
+            You can create several Active Directory instances by setting 
+            instance number larger than one. One primary Domain Controller 
+            and a few secondary DCs will be created.
+          attributeNames: units
+          minValue: 1
+          maxValue: 100
+          initial: 1
+          helpText: Enter an integer value between 1 and 100
+        - name: adminAccountName
+          type: string
+          label: Account Name
+          initial: Administrator
+          regexpValidator: '^[-\w]+$'
+          errorMessages:
+            invalid: 'Just letters, numbers, underscores and hyphens are allowed.'
+        - name: adminPassword
+          type: password
+          label: Administrator password
+          descriptionTitle: Passwords
+          description: >-
+            Windows requires strong password for service administration.
+            Your password should have at least one letter in each 
+            register, a number and a special character. Password length should be 
+            a minimum of 7 characters.
+    
+            Once you forget your password you won't be able to 
+            operate  the service until recovery password would be entered. So it's 
+            better for Recovery and Administrator password to be different.
+        - name: recoveryPassword
+          type: password
+          label: Recovery password
+          attributeNames: false
+        - name: assignFloatingIP
+          required: false
+          type: floatingip
+          label: Assign Floating IP
+          description: >-
+             Select to true to assign floating IP automatically to Primary DC
+          initial: false
+          required: false
+          widgetMedia:
+            css: {all: [muranodashboard/css/checkbox.css]}
+        - name: unitNamingPattern
+          type: string
+          label: Hostname template
+          description: >-
+            For your convenience all instance hostnames can be named
+            in the same way. Enter a name and use # character for incrementation. 
+            For example, host# turns into host1, host2, etc. Please follow Windows 
+            hostname restrictions.
+          required: false
+          regexpValidator: '^(([a-zA-Z0-9#][a-zA-Z0-9-#]*[a-zA-Z0-9#])\.)*([A-Za-z0-9#]|[A-Za-z0-9#][A-Za-z0-9-#]*[A-Za-z0-9#])$'
+          # FIXME: does not work for # turning into 2-digit numbers
+          maxLength: 15
+          helpText: Optional field for a machine hostname template
+          # temporaryHack
+          widgetMedia:
+            js: [muranodashboard/js/support_placeholder.js]
+            css: {all: [muranodashboard/css/support_placeholder.css]}
+      validators:
+        # if unitNamingPattern is given and dcInstances > 1, then '#' should occur in unitNamingPattern
+        - expr: {YAQL: $.serviceConfiguration.dcInstances < 2 or not $.serviceConfiguration.unitNamingPattern.bool() or '#' in $.serviceConfiguration.unitNamingPattern}
+          message: Incrementation symbol "#" is required in the Hostname template
+  - instanceConfiguration:
+      fields:
+        - name: title
+          type: string
+          required: false
+          hidden: true
+          attributeNames: false
+          descriptionTitle: Instance Configuration
+          description: Specify some instance parameters on which service would be created.
+        - name: flavor
+          type: flavor
+          label: Instance flavor
+          description: >-
+            Select registered in Openstack flavor. Consider that service performance 
+            depends on this parameter.
+          required: false
+        - name: osImage
+          type: image
+          imageType: windows
+          label: Instance image
+          description: >-
+            Select valid image for a service. Image should already be prepared and 
+            registered in glance.
+        - name: availabilityZone
+          type: azone
+          label: Availability zone
+          description: Select availability zone where service would be installed.
+          required: false

meta/packages/activedirectory/manifest.yaml

@@ -0,0 +1,16 @@
+Format: 1.0
+Type: Application
+FullName: io.murano.windows.activeDirectory.ActiveDirectory
+Name: Active Directory
+Description: |
+ A domain service hosted in Windows environment by using Active Directory Role.
+ May be clustered by combining a number of secondary domain controllers with one primary
+Author: 'murano.io'
+Tags: [Windows, Domain, demo, win2012, microsoft]
+Classes:
+ io.murano.windows.activeDirectory.ActiveDirectory: ad.yaml
+ io.murano.windows.activeDirectory.Controller: controller.yaml
+ io.murano.windows.activeDirectory.PrimaryController: primary_controller.yaml
+ io.murano.windows.activeDirectory.SecondaryController: secondary_controller.yaml
+# UI: ui.yaml     # default to ui.yaml, will use default if skipped
+Logo: logo2.png # defaults to logo.png, will use default if skipped

meta/packages/core/manifest.yaml

@@ -0,0 +1,12 @@
+Format: 1.0
+Type: Library
+FullName: io.murano.core
+Name: Core library
+Description: |
+ Core MuranoPL library
+Author: 'murano.io'
+Tags: [MuranoPL]
+Classes:
+ io.murano.Object: object.yaml
+ io.murano.Environment: environment.yaml
+ io.murano.Application: application.yaml

muranoapi/cmd/manage.py

@@ -21,13 +21,17 @@ import sys
 from oslo.config import cfg
 
 import muranoapi
+from muranoapi.db.catalog import api as db_catalog_api
 from muranoapi.db import session as db_session
-from muranoapi.openstack.common import log
+from muranoapi.openstack.common import log as logging
+from muranoapi.packages import application_package
 
 
 CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
 
 
+# TODO(ruhe): proper error handling
 def do_db_sync():
     """
     Place a database under migration control and upgrade,
@@ -36,12 +40,36 @@ def do_db_sync():
     db_session.db_sync()
 
 
+def _do_import_package(_dir):
+    LOG.info("Going to import Murano package from {0}".format(_dir))
+    pkg = application_package.load_from_dir(_dir)
+    result = db_catalog_api.package_upload(pkg.__dict__, None)
+    LOG.info("Finished import of package {0}".format(result.id))
+
+
+# TODO(ruhe): proper error handling
+def do_import_package():
+    """
+    Import Murano packages from local directories.
+    """
+    for _dir in CONF.command.directories:
+        _do_import_package(_dir)
+
+
 def add_command_parsers(subparsers):
-    parser = subparsers.add_parser('db_sync')
+    parser = subparsers.add_parser('db-sync')
     parser.set_defaults(func=do_db_sync)
     parser.add_argument('version', nargs='?')
     parser.add_argument('current_version', nargs='?')
 
+    parser = subparsers.add_parser('import-package')
+    parser.set_defaults(func=do_import_package)
+    parser.add_argument('directories',
+                        nargs='+',
+                        help="list of directories with Murano packages "
+                             "separated by space")
+
+
 command_opt = cfg.SubCommandOpt('command',
                                 title='Commands',
                                 help='Show available commands.',
@@ -55,8 +83,9 @@ def main():
         CONF(sys.argv[1:], project='murano-api', prog='murano-manage',
              version=muranoapi.__version__,
              default_config_files=default_config_files)
-        log.setup("murano-api")
+        logging.setup("murano-api")
     except RuntimeError as e:
+        LOG.error("murano-manage command failed: %s" % e)
         sys.exit("ERROR: %s" % e)
 
     try:

muranoapi/db/catalog/api.py

@@ -16,13 +16,16 @@ from sqlalchemy import or_
 from sqlalchemy import sql
 from webob import exc
 
-import muranoapi
 from muranoapi.db import models
 from muranoapi.db import session as db_session
 from muranoapi.openstack.common.gettextutils import _  # noqa
 from muranoapi.openstack.common import log as logging
 
-SEARCH_MAPPING = muranoapi.api.v1.SEARCH_MAPPING
+SEARCH_MAPPING = {'fqn': 'fully_qualified_name',
+                  'name': 'name',
+                  'created': 'created'
+                  }
+
 LOG = logging.getLogger(__name__)