diff --git a/examples/manifests/README.md b/examples/manifests/README.md new file mode 100644 index 00000000..e2746a0e --- /dev/null +++ b/examples/manifests/README.md @@ -0,0 +1,60 @@ +# Shipyard Manifests + +---- + +Shipyard manifests contain the examination of the payloads that the shipyard api will receive. +A complete manifest will consist of multiple yaml file's assembled in some way. Each yaml file will follow +Kubernetes style artifact definition. + +The high level expectation of what the data on this manifests will define is pictured here : + + + +---- + +## region_manifest.yaml + +Region is the largest resource shipyard can understand. +A region manifest will need to define : + +- Identity of the Region. Perhaps a name will suffice, but a UUID generated by shipyard might be applicable as well. +- Cloud : The type of cloud this region is running on. i.e. AIC, or AWS, or Google etc. +- deployOn : Whether the region UCP ( undercloud) is been deployed on VM's or Baremetal + +---- +## servers.yaml + +---- +## network.yaml + +---- +## hw_definition.yaml + +---- +## host_profile.yaml + +---- +## services.yaml + +Will define high level needs for all the services that need to run above the undercloud + +It relates to the files : + +## core_services.yaml +## clcp_services.yaml +## onap_services.yaml +## cdp_services.yaml + + +---- +## undercloud.yaml + +This file will incude the configuration aspects of the undercloud that are tunnables. +Such as : +i.e. +-Security +-RBAC definitions +-Certificates +-UCP Tunnables +-Kernel Tunnables, etc +-Agent Tunnables diff --git a/examples/manifests/hostprofile.yaml b/examples/manifests/hostprofile.yaml new file mode 100644 index 00000000..6ffb440d --- /dev/null +++ b/examples/manifests/hostprofile.yaml @@ -0,0 +1,151 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#################### +# +# bootstrap_seed.yaml - Site server design definition for physical layer +# +#################### +# version the schema in this file so consumers can rationally parse it + +--- +apiVersion: 'v1.0' +kind: HostProfile +metadata: + name: default + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces + # No magic to this host_profile, it just provides a way to specify + # sitewide settings. If it is absent from a node's inheritance chain + # then these values will NOT be applied +spec: + # OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such + # as IPMI over vender-specific when possible. + oob: + type: ipmi + # OOB networking should be preconfigured, but we can include a network + # definition for validation or enhancement (DNS registration) + network: oob + account: admin + credential: admin + # Specify storage layout of base OS. Ceph out of scope + storage: + # How storage should be carved up: lvm (logical volumes), flat + # (single partition) + layout: lvm + # Info specific to the boot and root disk/partitions + bootdisk: + # Device will specify an alias defined in hwdefinition.yaml + device: primary_boot + # For LVM, the size of the partition added to VG as a PV + # For flat, the size of the partition formatted as ext4 + root_size: 50g + # The /boot partition. If not specified, /boot will in root + boot_size: 2g + # Info for additional partitions. Need to balance between + # flexibility and complexity + partitions: + - name: logs + device: primary_boot + # Partition uuid if needed + part_uuid: 84db9664-f45e-11e6-823d-080027ef795a + size: 10g + # Optional, can carve up unformatted block devices + mountpoint: /var/log + fstype: ext4 + mount_options: defaults + # Filesystem UUID or label can be specified. UUID recommended + fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e + fs_label: logs + # Platform (Operating System) settings + platform: + image: ubuntu_16.04_hwe + kernel_params: default + # Additional metadata to apply to a node + metadata: + # Base URL of the introspection service - may go in curtin data + introspection_url: http://172.16.1.10:9090 +--- +apiVersion: 'v1.0' +kind: HostProfile +metadata: + name: k8-node + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + # host_profile inheritance allows for deduplication of common CIs + # Inheritance is additive for CIs that are lists of multiple items + # To remove an inherited list member, prefix the primary key value + # with '!'. + host_profile: defaults + # Hardware profile will map hardware specific details to the abstract + # names uses in the host profile as well as specify hardware specific + # configs. A viable model should be to build a host profile without a + # hardware_profile and then for each node inherit the host profile and + # specify a hardware_profile to map that node's hardware to the abstract + # settings of the host_profile + hardware_profile: HPGen9v3 + # Network interfaces. + interfaces: + # Keyed on device_name + # pxe is a special marker indicating which device should be used for pxe boot + - device_name: pxe + # The network link attached to this + network_link: pxe + # Slaves will specify aliases from hwdefinition.yaml + slaves: + - prim_nic01 + # Which networks will be configured on this interface + networks: + - name: pxe + - device_name: bond0 + network_link: gp + # If multiple slaves are specified, but no bonding config + # is applied to the link, design validation will fail + slaves: + - prim_nic01 + - prim_nic02 + # If multiple networks are specified, but no trunking + # config is applied to the link, design validation will fail + networks: + - name: mgmt + - name: private + metadata: + # Explicit tag assignment + tags: + - 'test' + # MaaS supports key/value pairs. Not sure of the use yet + owner_data: + foo: bar +--- +apiVersion: 'v1.0' +kind: HostProfile +metadata: + name: k8-node-public + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + host_profile: k8-node + interfaces: + - device_name: bond0 + networks: + # This is additive, so adds a network to those defined in the host_profile + # inheritance chain + - name: public +--- \ No newline at end of file diff --git a/examples/manifests/hwdefinition.yaml b/examples/manifests/hwdefinition.yaml new file mode 100644 index 00000000..d7daa741 --- /dev/null +++ b/examples/manifests/hwdefinition.yaml @@ -0,0 +1,58 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################# +# +# bootstrap_hwdefinition.yaml - Definitions of server hardware layout +# +############################################################################# +# version the schema in this file so consumers can rationally parse it +--- +apiVersion: 'v1.0' +kind: HardwareProfile +metadata: + name: HPGen8v3 + region: sitename + date: 17-FEB-2017 + description: Sample hardware definition + author: Scott Hussey +spec: + # Vendor of the server chassis + vendor: HP + # Generation of the chassis model + generation: '8' + # Version of the chassis model within its generation - not version of the hardware definition + hw_version: '3' + # The certified version of the chassis BIOS + bios_version: '2.2.3' + # Mode of the default boot of hardware - bios, uefi + boot_mode: bios + # Protocol of boot of the hardware - pxe, usb, hdd + bootstrap_protocol: pxe + # Which interface to use for network booting within the OOB manager, not OS device + pxe_interface: 0 + # Map hardware addresses to aliases/roles to allow a mix of hardware configs + # in a site to result in a consistent configuration + device_aliases: + pci: + - address: pci@0000:00:03.0 + alias: prim_nic01 + # type could identify expected hardware - used for hardware manifest validation + type: '82540EM Gigabit Ethernet Controller' + - address: pci@0000:00:04.0 + alias: prim_nic02 + type: '82540EM Gigabit Ethernet Controller' + scsi: + - address: scsi@2:0.0.0 + alias: primary_boot + type: 'VBOX HARDDISK' \ No newline at end of file diff --git a/examples/manifests/manifest_hierarchy.png b/examples/manifests/manifest_hierarchy.png new file mode 100644 index 00000000..b571f21f Binary files /dev/null and b/examples/manifests/manifest_hierarchy.png differ diff --git a/examples/manifests/network.yml b/examples/manifests/network.yml new file mode 100644 index 00000000..26c9d052 --- /dev/null +++ b/examples/manifests/network.yml @@ -0,0 +1,230 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#################### +# +# network.yaml - Network infor,ation design definition for physical layer +# +#################### +# version the schema in this file so consumers can rationally parse it + +--- +--- +apiVersion: 'v1.0' +kind: NetworkLink +metadata: + name: oob + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on +spec: + bonding: + mode: none + mtu: 1500 + linkspeed: 100full + trunking: + mode: none + default_network: oob +--- +# pxe is a bit of 'magic' indicating the link config used when PXE booting +# a node. All other links indicate network configs applied when the node +# is deployed. +apiVersion: 'v1.0' +kind: NetworkLink +metadata: + name: pxe + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on +spec: + bonding: + mode: none + mtu: 1500 + linkspeed: auto + # Is this link supporting multiple layer 2 networks? + # none is a port-based VLAN identified by default_network + # tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr + trunking: + mode: none + # use name, will translate to VLAN ID + default_network: pxe +--- +apiVersion: 'v1.0' +kind: NetworkLink +metadata: + name: gp + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on + # pxe is a bit of 'magic' indicating the link config used when PXE booting + # a node. All other links indicate network configs applied when the node + # is deployed. +spec: + # If this link is a bond of physical links, how is it configured + # 802.3ad + # active-backup + # balance-rr + # Can add support for others down the road + bonding: + mode: 802.3ad + # For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4 + hash: layer3+4 + # 802.3ad specific options + peer_rate: slow + mon_rate: default + up_delay: default + down_delay: default + mtu: 9000 + linkspeed: auto + # Is this link supporting multiple layer 2 networks? + trunking: + mode: tagged + default_network: mgmt +--- +apiVersion: 'v1.0' +kind: Network +metadata: + name: oob + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + allocation: static + cidr: 172.16.100.0/24 + ranges: + - type: static + start: 172.16.100.15 + end: 172.16.100.254 + dns: + domain: ilo.sitename.att.com + servers: 172.16.100.10 +--- +apiVersion: 'v1.0' +kind: Network +metadata: + name: pxe + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + # Layer 2 VLAN segment id, could support other segmentations. Optional + vlan_id: '99' + # How are addresses assigned? + allocation: dhcp + # MTU for this VLAN interface, if not specified it will be inherited from the link + mtu: 1500 + # Network address + cidr: 172.16.0.0/24 + # Desribe IP address ranges + ranges: + - type: dhcp + start: 172.16.0.5 + end: 172.16.0.254 + # DNS settings for this network + dns: + # Domain addresses on this network will be registered under + domain: admin.sitename.att.com + # DNS servers that a server using this network as its default gateway should use + servers: 172.16.0.10 +--- +apiVersion: 'v1.0' +kind: Network +metadata: + name: mgmt + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '100' + # How are addresses assigned? + allocation: static + # Allow MTU to be inherited from link the network rides on + mtu: 1500 + # Network address + cidr: 172.16.1.0/24 + # Desribe IP address ranges + ranges: + - type: static + start: 172.16.1.15 + end: 172.16.1.254 + # Static routes to be added for this network + routes: + - subnet: 0.0.0.0/0 + # A blank gateway would leave to a static route specifying + # only the interface as a source + gateway: 172.16.1.1 + metric: 10 + # DNS settings for this network + dns: + # Domain addresses on this network will be registered under + domain: mgmt.sitename.example.com + # DNS servers that a server using this network as its default gateway should use + servers: 172.16.1.9,172.16.1.10 +--- +apiVersion: 'v1.0' +kind: Network +metadata: + name: private + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '101' + allocation: static + mtu: 9000 + cidr: 172.16.2.0/24 + # Desribe IP address ranges + ranges: + # Type can be reserved (not used for baremetal), static (all explicit + # assignments should fall here), dhcp (will be used by a DHCP server on this network) + - type: static + start: 172.16.2.15 + end: 172.16.2.254 + dns: + domain: priv.sitename.example.com + servers: 172.16.2.9,172.16.2.10 +--- +apiVersion: 'v1.0' +kind: Network +metadata: + name: public + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '102' + # How are addresses assigned? + allocation: static + # MTU size for the VLAN interface + mtu: 1500 + cidr: 172.16.3.0/24 + # Desribe IP address ranges + ranges: + - type: static + start: 172.16.3.15 + end: 172.16.3.254 + routes: + - subnet: 0.0.0.0/0 + gateway: 172.16.3.1 + metric: 9 + dns: + domain: sitename.example.com + servers: 8.8.8.8 \ No newline at end of file diff --git a/examples/manifests/region_manifest.yml b/examples/manifests/region_manifest.yml new file mode 100644 index 00000000..db6ac399 --- /dev/null +++ b/examples/manifests/region_manifest.yml @@ -0,0 +1,60 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#################### +# +# region_manifest.yaml - Region Manifest File , encapsulates the multiple files +# +#################### +# version + +--- +# +# This describes the Global details of a Region +# +apiVersion: 'v1.0' +kind: Region +metadata: + name: sitename + date: 17-FEB-2017 + description: Sample site design + author: sh8121@att.com +spec: + + +------- +imports: + # Servers will include the list of Servers + # For Each Server it includes + # information such as : + # # OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such + # as IPMI over vender-specific when possible. + # oob: + # type: ipmi + # OOB networking should be preconfigured, but we can include a network + # definition for validation or enhancement (DNS registration) + # Specify storage layout of base OS. Ceph out of scope + # storage: + # How storage should be carved up: lvm (logical volumes), flat + # (single partition) + # Platform (Operating System) settings + # platform: + # Additional metadata to apply to a node + @ metadata: + - 'servers.yaml' + + - 'network.yaml' + - 'hwdefinition.yaml' + - 'hostprofile.yaml' + + \ No newline at end of file diff --git a/examples/manifests/servers.yaml b/examples/manifests/servers.yaml new file mode 100644 index 00000000..360075f9 --- /dev/null +++ b/examples/manifests/servers.yaml @@ -0,0 +1,420 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#################### +# +# bootstrap_seed.yaml - Site server design definition for physical layer +# +#################### +# version the schema in this file so consumers can rationally parse it +--- +apiVersion: 'v1.0' +kind: Region +metadata: + name: sitename + date: 17-FEB-2017 + description: Sample site design + author: sh8121@att.com +spec: + # Not sure if we have site wide data that doesn't fall into another 'Kind' +--- +apiVersion: 'v1.0' +kind: NetworkLink +metadata: + name: oob + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on +spec: + bonding: + mode: none + mtu: 1500 + linkspeed: 100full + trunking: + mode: none + default_network: oob +--- +# pxe is a bit of 'magic' indicating the link config used when PXE booting +# a node. All other links indicate network configs applied when the node +# is deployed. +apiVersion: 'v1.0' +kind: NetworkLink +metadata: + name: pxe + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on +spec: + bonding: + mode: none + mtu: 1500 + linkspeed: auto + # Is this link supporting multiple layer 2 networks? + # none is a port-based VLAN identified by default_network + # tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr + trunking: + mode: none + # use name, will translate to VLAN ID + default_network: pxe +--- +apiVersion: 'v1.0' +kind: NetworkLink +metadata: + name: gp + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on + # pxe is a bit of 'magic' indicating the link config used when PXE booting + # a node. All other links indicate network configs applied when the node + # is deployed. +spec: + # If this link is a bond of physical links, how is it configured + # 802.3ad + # active-backup + # balance-rr + # Can add support for others down the road + bonding: + mode: 802.3ad + # For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4 + hash: layer3+4 + # 802.3ad specific options + peer_rate: slow + mon_rate: default + up_delay: default + down_delay: default + mtu: 9000 + linkspeed: auto + # Is this link supporting multiple layer 2 networks? + trunking: + mode: tagged + default_network: mgmt +--- +apiVersion: 'v1.0' +kind: Network +metadata: + name: oob + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + allocation: static + cidr: 172.16.100.0/24 + ranges: + - type: static + start: 172.16.100.15 + end: 172.16.100.254 + dns: + domain: ilo.sitename.att.com + servers: 172.16.100.10 +--- +apiVersion: 'v1.0' +kind: Network +metadata: + name: pxe + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + # Layer 2 VLAN segment id, could support other segmentations. Optional + vlan_id: '99' + # How are addresses assigned? + allocation: dhcp + # MTU for this VLAN interface, if not specified it will be inherited from the link + mtu: 1500 + # Network address + cidr: 172.16.0.0/24 + # Desribe IP address ranges + ranges: + - type: dhcp + start: 172.16.0.5 + end: 172.16.0.254 + # DNS settings for this network + dns: + # Domain addresses on this network will be registered under + domain: admin.sitename.att.com + # DNS servers that a server using this network as its default gateway should use + servers: 172.16.0.10 +--- +apiVersion: 'v1.0' +kind: Network +metadata: + name: mgmt + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '100' + # How are addresses assigned? + allocation: static + # Allow MTU to be inherited from link the network rides on + mtu: 1500 + # Network address + cidr: 172.16.1.0/24 + # Desribe IP address ranges + ranges: + - type: static + start: 172.16.1.15 + end: 172.16.1.254 + # Static routes to be added for this network + routes: + - subnet: 0.0.0.0/0 + # A blank gateway would leave to a static route specifying + # only the interface as a source + gateway: 172.16.1.1 + metric: 10 + # DNS settings for this network + dns: + # Domain addresses on this network will be registered under + domain: mgmt.sitename.example.com + # DNS servers that a server using this network as its default gateway should use + servers: 172.16.1.9,172.16.1.10 +--- +apiVersion: 'v1.0' +kind: Network +metadata: + name: private + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '101' + allocation: static + mtu: 9000 + cidr: 172.16.2.0/24 + # Desribe IP address ranges + ranges: + # Type can be reserved (not used for baremetal), static (all explicit + # assignments should fall here), dhcp (will be used by a DHCP server on this network) + - type: static + start: 172.16.2.15 + end: 172.16.2.254 + dns: + domain: priv.sitename.example.com + servers: 172.16.2.9,172.16.2.10 +--- +apiVersion: 'v1.0' +kind: Network +metadata: + name: public + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + vlan_id: '102' + # How are addresses assigned? + allocation: static + # MTU size for the VLAN interface + mtu: 1500 + cidr: 172.16.3.0/24 + # Desribe IP address ranges + ranges: + - type: static + start: 172.16.3.15 + end: 172.16.3.254 + routes: + - subnet: 0.0.0.0/0 + gateway: 172.16.3.1 + metric: 9 + dns: + domain: sitename.example.com + servers: 8.8.8.8 +--- +apiVersion: 'v1.0' +kind: HostProfile +metadata: + name: default + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces + # No magic to this host_profile, it just provides a way to specify + # sitewide settings. If it is absent from a node's inheritance chain + # then these values will NOT be applied +spec: + # OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such + # as IPMI over vender-specific when possible. + oob: + type: ipmi + # OOB networking should be preconfigured, but we can include a network + # definition for validation or enhancement (DNS registration) + network: oob + account: admin + credential: admin + # Specify storage layout of base OS. Ceph out of scope + storage: + # How storage should be carved up: lvm (logical volumes), flat + # (single partition) + layout: lvm + # Info specific to the boot and root disk/partitions + bootdisk: + # Device will specify an alias defined in hwdefinition.yaml + device: primary_boot + # For LVM, the size of the partition added to VG as a PV + # For flat, the size of the partition formatted as ext4 + root_size: 50g + # The /boot partition. If not specified, /boot will in root + boot_size: 2g + # Info for additional partitions. Need to balance between + # flexibility and complexity + partitions: + - name: logs + device: primary_boot + # Partition uuid if needed + part_uuid: 84db9664-f45e-11e6-823d-080027ef795a + size: 10g + # Optional, can carve up unformatted block devices + mountpoint: /var/log + fstype: ext4 + mount_options: defaults + # Filesystem UUID or label can be specified. UUID recommended + fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e + fs_label: logs + # Platform (Operating System) settings + platform: + image: ubuntu_16.04_hwe + kernel_params: default + # Additional metadata to apply to a node + metadata: + # Base URL of the introspection service - may go in curtin data + introspection_url: http://172.16.1.10:9090 +--- +apiVersion: 'v1.0' +kind: HostProfile +metadata: + name: k8-node + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + # host_profile inheritance allows for deduplication of common CIs + # Inheritance is additive for CIs that are lists of multiple items + # To remove an inherited list member, prefix the primary key value + # with '!'. + host_profile: defaults + # Hardware profile will map hardware specific details to the abstract + # names uses in the host profile as well as specify hardware specific + # configs. A viable model should be to build a host profile without a + # hardware_profile and then for each node inherit the host profile and + # specify a hardware_profile to map that node's hardware to the abstract + # settings of the host_profile + hardware_profile: HPGen9v3 + # Network interfaces. + interfaces: + # Keyed on device_name + # pxe is a special marker indicating which device should be used for pxe boot + - device_name: pxe + # The network link attached to this + network_link: pxe + # Slaves will specify aliases from hwdefinition.yaml + slaves: + - prim_nic01 + # Which networks will be configured on this interface + networks: + - name: pxe + - device_name: bond0 + network_link: gp + # If multiple slaves are specified, but no bonding config + # is applied to the link, design validation will fail + slaves: + - prim_nic01 + - prim_nic02 + # If multiple networks are specified, but no trunking + # config is applied to the link, design validation will fail + networks: + - name: mgmt + - name: private + metadata: + # Explicit tag assignment + tags: + - 'test' + # MaaS supports key/value pairs. Not sure of the use yet + owner_data: + foo: bar +--- +apiVersion: 'v1.0' +kind: HostProfile +metadata: + name: k8-node-public + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + host_profile: k8-node + interfaces: + - device_name: bond0 + networks: + # This is additive, so adds a network to those defined in the host_profile + # inheritance chain + - name: public +--- +apiVersion: 'v1.0' +kind: BaremetalNode +metadata: + name: controller01 + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + host_profile: k8-node-public + # the hostname for a server, could be used in multiple DNS domains to + # represent different interfaces + interfaces: + - device_name: bond0 + networks: + # '!' prefix for the value of the primary key indicates a record should be removed + - name: '!private' + # Addresses assigned to network interfaces + addressing: + # Which network the address applies to. If a network appears in addressing + # that isn't assigned to an interface, design validation will fail + - network: pxe + # The address assigned. Either a explicit IPv4 or IPv6 address + # or dhcp or slaac + address: dhcp + - network: mgmt + address: 172.16.1.20 + - network: public + address: 172.16.3.20 + metadata: + tags: + - os_ctl + rack: rack01 +--- +apiVersion: 'v1.0' +kind: BaremetalNode +metadata: + name: compute01 + region: sitename + date: 17-FEB-2017 + author: sh8121@att.com + description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces +spec: + host_profile: k8-node + addressing: + - network: pxe + address: dhcp + - network: mgmt + address: 172.16.1.21 + - network: private + address: 172.16.2.21 diff --git a/examples/manifests/services.yaml b/examples/manifests/services.yaml new file mode 100644 index 00000000..358010c6 --- /dev/null +++ b/examples/manifests/services.yaml @@ -0,0 +1,27 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################# +# +# services.yaml - Definitions of server hardware layout +# +############################################################################# +# version the schema in this file so consumers can rationally parse it +--- +# +# Is this where I include a list of files per service ? +# +# +# Assuming something like this works for the insertion + +imports: \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..85e81e05 --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup + +setup(name='shipyard_airflow', + version='0.1a1', + description='API for managing Airflow-based orchestration', + url='http://github.com/att-comdev/shipyard', + author='Anthony Lin - AT&T', + author_email='al498u@att.com', + license='Apache 2.0', + packages=['shipyard_airflow', + 'shipyard_airflow.control'], + install_requires=[ + 'falcon', + 'requests', + 'configparser', + 'uwsgi>1.4' + ] + ) + diff --git a/shipyard_airflow/README.md b/shipyard_airflow/README.md new file mode 100644 index 00000000..88496910 --- /dev/null +++ b/shipyard_airflow/README.md @@ -0,0 +1,12 @@ +## Shipyard Airflow ## + +A python REST workflow orchestrator + +To run: + +``` +$ virtualenv -p python3 /var/tmp/shipyard +$ . /var/tmp/shipyard/bin/activate +$ python setup.py install +$ uwsgi --http :9000 -w shipyard_airflow.shipyard --callable shipyard -L +``` diff --git a/shipyard_airflow/__init__.py b/shipyard_airflow/__init__.py new file mode 100644 index 00000000..2a385a45 --- /dev/null +++ b/shipyard_airflow/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/shipyard_airflow/control/__init__.py b/shipyard_airflow/control/__init__.py new file mode 100644 index 00000000..f10bbbf6 --- /dev/null +++ b/shipyard_airflow/control/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/shipyard_airflow/control/api.py b/shipyard_airflow/control/api.py new file mode 100644 index 00000000..4a004aa8 --- /dev/null +++ b/shipyard_airflow/control/api.py @@ -0,0 +1,54 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import json + +from .regions import RegionsResource, RegionResource +from .base import ShipyardRequest, BaseResource +from .tasks import TaskResource +from .dag_runs import DagRunResource +from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware + +def start_api(): + + control_api = falcon.API(request_type=ShipyardRequest, + middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()]) + + control_api.add_route('/versions', VersionsResource()) + + # v1.0 of Shipyard API + v1_0_routes = [ + # API for managing region data + ('/regions', RegionsResource()), + ('/regions/{region_id}', RegionResource()), + ('/dags/{dag_id}/tasks/{task_id}', TaskResource()), + ('/dags/{dag_id}/dag_runs', DagRunResource()), + ] + + for path, res in v1_0_routes: + control_api.add_route('/api/experimental' + path, res) + + return control_api + +class VersionsResource(BaseResource): + + authorized_roles = ['anyone'] + + def on_get(self, req, resp): + resp.body = json.dumps({'v1.0': { + 'path': '/api/v1.0', + 'status': 'stable' + }}) + resp.status = falcon.HTTP_200 + diff --git a/shipyard_airflow/control/base.py b/shipyard_airflow/control/base.py new file mode 100644 index 00000000..05912178 --- /dev/null +++ b/shipyard_airflow/control/base.py @@ -0,0 +1,111 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon, falcon.request as request +import uuid +import json +import configparser +import os + +class BaseResource(object): + + authorized_roles = [] + + def on_options(self, req, resp): + self_attrs = dir(self) + methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'] + allowed_methods = [] + + for m in methods: + if 'on_' + m.lower() in self_attrs: + allowed_methods.append(m) + + resp.headers['Allow'] = ','.join(allowed_methods) + resp.status = falcon.HTTP_200 + + # By default, no one is authorized to use a resource + def authorize_roles(self, role_list): + authorized = set(self.authorized_roles) + applied = set(role_list) + + if authorized.isdisjoint(applied): + return False + else: + return True + + # Error Handling + def return_error(self, resp, status_code, message="", retry=False): + """ + Write a error message body and throw a Falcon exception to trigger an HTTP status + + :param resp: Falcon response object to update + :param status_code: Falcon status_code constant + :param message: Optional error message to include in the body + :param retry: Optional flag whether client should retry the operation. Can ignore if we rely solely on 4XX vs 5xx status codes + """ + resp.body = json.dumps({'type': 'error', 'message': message, 'retry': retry}) + resp.content_type = 'application/json' + resp.status = status_code + + # Get Config Data + def retrieve_config(self, section="", data=""): + + # The current assumption is that shipyard.conf will be placed in a fixed path + # within the shipyard container - Path TBD + path = '/home/ubuntu/att-comdev/shipyard/shipyard_airflow/control/shipyard.conf' + + # Check that shipyard.conf exists + if os.path.isfile(path): + config = configparser.ConfigParser() + config.read(path) + + # Retrieve data from shipyard.conf + query_data = config.get(section, data) + + return query_data + else: + return 'Error - Missing Configuration File' + + +class ShipyardRequestContext(object): + + def __init__(self): + self.log_level = 'error' + self.user = None + self.roles = ['anyone'] + self.request_id = str(uuid.uuid4()) + self.external_marker = None + + def set_log_level(self, level): + if level in ['error', 'info', 'debug']: + self.log_level = level + + def set_user(self, user): + self.user = user + + def add_role(self, role): + self.roles.append(role) + + def add_roles(self, roles): + self.roles.extend(roles) + + def remove_role(self, role): + self.roles = [x for x in self.roles + if x != role] + + def set_external_marker(self, marker): + self.external_marker = str(marker)[:32] + +class ShipyardRequest(request.Request): + context_type = ShipyardRequestContext + diff --git a/shipyard_airflow/control/dag_runs.py b/shipyard_airflow/control/dag_runs.py new file mode 100644 index 00000000..7a462d3d --- /dev/null +++ b/shipyard_airflow/control/dag_runs.py @@ -0,0 +1,47 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import requests +import json + +from .base import BaseResource + +class DagRunResource(BaseResource): + + authorized_roles = ['user'] + + def on_post(self, req, resp, dag_id, run_id=None, conf=None, execution_date=None): + # Retrieve URL + web_server_url = self.retrieve_config('BASE', 'WEB_SERVER') + + if 'Error' in web_server_url: + resp.status = falcon.HTTP_400 + resp.body = json.dumps({'Error': 'Missing Configuration File'}) + return + else: + req_url = '{}/api/experimental/dags/{}/dag_runs'.format(web_server_url, dag_id) + + response = requests.post(req_url, + json={ + "run_id": run_id, + "conf": conf, + "execution_date": execution_date, + }) + + if response.ok: + resp.status = falcon.HTTP_200 + else: + self.return_error(resp, falcon.HTTP_400, 'Fail to Execute Dag') + return + diff --git a/shipyard_airflow/control/middleware.py b/shipyard_airflow/control/middleware.py new file mode 100644 index 00000000..c4db2c76 --- /dev/null +++ b/shipyard_airflow/control/middleware.py @@ -0,0 +1,89 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import logging + +class AuthMiddleware(object): + + # Authentication + def process_request(self, req, resp): + ctx = req.context + token = req.get_header('X-Auth-Token') + + user = self.validate_token(token) + + if user is not None: + ctx.set_user(user) + user_roles = self.role_list(user) + ctx.add_roles(user_roles) + else: + ctx.add_role('anyone') + + # Authorization + def process_resource(self, req, resp, resource, params): + ctx = req.context + + if not resource.authorize_roles(ctx.roles): + raise falcon.HTTPUnauthorized('Authentication required', + ('This resource requires an authorized role.')) + + # Return the username associated with an authenticated token or None + def validate_token(self, token): + if token == '10': + return 'shipyard' + elif token == 'admin': + return 'admin' + else: + return None + + # Return the list of roles assigned to the username + # Roles need to be an enum + def role_list(self, username): + if username == 'shipyard': + return ['user'] + elif username == 'admin': + return ['user', 'admin'] + +class ContextMiddleware(object): + + def process_request(self, req, resp): + ctx = req.context + + requested_logging = req.get_header('X-Log-Level') + + if requested_logging == 'DEBUG' and 'admin' in ctx.roles: + ctx.set_log_level('debug') + elif requested_logging == 'INFO': + ctx.set_log_level('info') + + ext_marker = req.get_header('X-Context-Marker') + ctx.set_external_marker(ext_marker if ext_marker is not None else '') + +class LoggingMiddleware(object): + + def __init__(self): + self.logger = logging.getLogger('shipyard.control') + + def process_response(self, req, resp, resource, req_succeeded): + ctx = req.context + + extra = { + 'user': ctx.user, + 'req_id': ctx.request_id, + 'external_ctx': ctx.external_marker, + } + + resp.append_header('X-Shipyard-Req', ctx.request_id) + self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra) + diff --git a/shipyard_airflow/control/regions.py b/shipyard_airflow/control/regions.py new file mode 100644 index 00000000..ac6d970a --- /dev/null +++ b/shipyard_airflow/control/regions.py @@ -0,0 +1,31 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon + +from .base import BaseResource + +class RegionsResource(BaseResource): + + authorized_roles = ['user'] + + def on_get(self, req, resp): + resp.status = falcon.HTTP_200 + +class RegionResource(BaseResource): + + authorized_roles = ['user'] + + def on_get(self, req, resp, region_id): + resp.status = falcon.HTTP_200 + diff --git a/shipyard_airflow/control/shipyard.conf b/shipyard_airflow/control/shipyard.conf new file mode 100644 index 00000000..6dadc48c --- /dev/null +++ b/shipyard_airflow/control/shipyard.conf @@ -0,0 +1,3 @@ +[BASE] +WEB_SERVER=http://localhost:32080 + diff --git a/shipyard_airflow/control/tasks.py b/shipyard_airflow/control/tasks.py new file mode 100644 index 00000000..9cb2f6e8 --- /dev/null +++ b/shipyard_airflow/control/tasks.py @@ -0,0 +1,43 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import json +import requests + +from .base import BaseResource + +class TaskResource(BaseResource): + + authorized_roles = ['user'] + + def on_get(self, req, resp, dag_id, task_id): + # Retrieve URL + web_server_url = self.retrieve_config('BASE', 'WEB_SERVER') + + if 'Error' in web_server_url: + resp.status = falcon.HTTP_400 + resp.body = json.dumps({'Error': 'Missing Configuration File'}) + return + else: + req_url = '{}/api/experimental/dags/{}/tasks/{}'.format(web_server_url, dag_id, task_id) + task_details = requests.get(req_url).json() + + if 'error' in task_details: + resp.status = falcon.HTTP_400 + resp.body = json.dumps(task_details) + return + else: + resp.status = falcon.HTTP_200 + resp.body = json.dumps(task_details) + diff --git a/shipyard_airflow/setup.py b/shipyard_airflow/setup.py new file mode 100644 index 00000000..85e81e05 --- /dev/null +++ b/shipyard_airflow/setup.py @@ -0,0 +1,33 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup + +setup(name='shipyard_airflow', + version='0.1a1', + description='API for managing Airflow-based orchestration', + url='http://github.com/att-comdev/shipyard', + author='Anthony Lin - AT&T', + author_email='al498u@att.com', + license='Apache 2.0', + packages=['shipyard_airflow', + 'shipyard_airflow.control'], + install_requires=[ + 'falcon', + 'requests', + 'configparser', + 'uwsgi>1.4' + ] + ) + diff --git a/shipyard_airflow/shipyard.py b/shipyard_airflow/shipyard.py new file mode 100644 index 00000000..887028b3 --- /dev/null +++ b/shipyard_airflow/shipyard.py @@ -0,0 +1,40 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import control.api as api + +def start_shipyard(): + + # Setup root logger + logger = logging.getLogger('shipyard') + + logger.setLevel('DEBUG') + ch = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + + # Specalized format for API logging + logger = logging.getLogger('shipyard.control') + logger.propagate = False + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s') + + ch = logging.StreamHandler() + ch.setFormatter(formatter) + logger.addHandler(ch) + + return api.start_api() + +shipyard = start_shipyard() +